From 021d4de36cecc157393f4ac838627e23fab0ee88 Mon Sep 17 00:00:00 2001 From: Di Xu Date: Tue, 1 Aug 2017 23:17:44 +0800 Subject: [PATCH] run nsenter in host namespace for containerized kubelet --- pkg/volume/host_path/host_path.go | 29 ++-- pkg/volume/host_path/nsenter.go | 150 ++++++++++++++++++++ pkg/volume/host_path/nsenter_unsupported.go | 66 +++++++++ 3 files changed, 237 insertions(+), 8 deletions(-) create mode 100644 pkg/volume/host_path/nsenter.go create mode 100644 pkg/volume/host_path/nsenter_unsupported.go diff --git a/pkg/volume/host_path/host_path.go b/pkg/volume/host_path/host_path.go index e00fc8e5125..0b5f4b906a6 100644 --- a/pkg/volume/host_path/host_path.go +++ b/pkg/volume/host_path/host_path.go @@ -107,7 +107,7 @@ func (plugin *hostPathPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts vo path := hostPathVolumeSource.Path return &hostPathMounter{ - hostPath: &hostPath{path: path, pathType: hostPathVolumeSource.Type}, + hostPath: &hostPath{path: path, pathType: hostPathVolumeSource.Type, containerized: opts.Containerized}, readOnly: readOnly, }, nil } @@ -176,8 +176,9 @@ func newProvisioner(options volume.VolumeOptions, host volume.VolumeHost, plugin // HostPath volumes represent a bare host file or directory mount. // The direct at the specified path will be directly exposed to the container. type hostPath struct { - path string - pathType *v1.HostPathType + path string + pathType *v1.HostPathType + containerized bool volume.MetricsNil } @@ -217,7 +218,7 @@ func (b *hostPathMounter) SetUp(fsGroup *int64) error { if *b.pathType == v1.HostPathUnset { return nil } - return checkType(b.GetPath(), b.pathType) + return checkType(b.GetPath(), b.pathType, b.containerized) } // SetUpAt does not make sense for host paths - probably programmer error. @@ -441,10 +442,22 @@ func newOSFileTypeChecker(path string, checker fileTypeChecker) (hostPathTypeChe return &ftc, nil } -func checkType(path string, pathType *v1.HostPathType) error { - ftc, err := newOSFileTypeChecker(path, &defaultFileTypeChecker{}) - if err != nil { - return err +func checkType(path string, pathType *v1.HostPathType, containerized bool) error { + var ftc hostPathTypeChecker + var err error + if containerized { + // For a containerized kubelet, use nsenter to run commands in + // the host's mount namespace. + // TODO(dixudx): setns into docker's mount namespace, and then run the exact same go code for checks/setup + ftc, err = newNsenterFileTypeChecker(path) + if err != nil { + return err + } + } else { + ftc, err = newOSFileTypeChecker(path, &defaultFileTypeChecker{}) + if err != nil { + return err + } } return checkTypeInternal(ftc, pathType) } diff --git a/pkg/volume/host_path/nsenter.go b/pkg/volume/host_path/nsenter.go new file mode 100644 index 00000000000..c3c3cfece57 --- /dev/null +++ b/pkg/volume/host_path/nsenter.go @@ -0,0 +1,150 @@ +// +build linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package host_path + +import ( + "fmt" + + "k8s.io/utils/exec" +) + +const ( + hostProcMountsNamespace = "/rootfs/proc/1/ns/mnt" + nsenterCmd = "nsenter" + statCmd = "stat" + touchCmd = "touch" + mkdirCmd = "mkdir" +) + +// nsenterFileTypeChecker is part of experimental support for running the kubelet +// in a container. nsenterFileTypeChecker works by executing "nsenter" to run commands in +// the host's mount namespace. +// +// nsenterFileTypeChecker requires: +// +// 1. The host's root filesystem must be available at "/rootfs"; +// 2. The "nsenter" binary must be on the Kubelet process' PATH in the container's +// filesystem; +// 3. The Kubelet process must have CAP_SYS_ADMIN (required by "nsenter"); at +// the present, this effectively means that the kubelet is running in a +// privileged container; +// 4. The host image must have "stat", "touch", "mkdir" binaries in "/bin", "/usr/sbin", or "/usr/bin"; + +type nsenterFileTypeChecker struct { + path string + exists bool +} + +func newNsenterFileTypeChecker(path string) (hostPathTypeChecker, error) { + ftc := &nsenterFileTypeChecker{path: path} + ftc.Exists() + return ftc, nil +} + +func (ftc *nsenterFileTypeChecker) Exists() bool { + args := []string{ + fmt.Sprintf("--mount=%s", hostProcMountsNamespace), + "--", + "ls", + ftc.path, + } + exec := exec.New() + _, err := exec.Command(nsenterCmd, args...).CombinedOutput() + if err == nil { + ftc.exists = true + } + return ftc.exists +} + +func (ftc *nsenterFileTypeChecker) IsFile() bool { + if !ftc.Exists() { + return false + } + return !ftc.IsDir() +} + +func (ftc *nsenterFileTypeChecker) MakeFile() error { + args := []string{ + fmt.Sprintf("--mount=%s", hostProcMountsNamespace), + "--", + touchCmd, + ftc.path, + } + exec := exec.New() + if _, err := exec.Command(nsenterCmd, args...).CombinedOutput(); err != nil { + return err + } + return nil +} + +func (ftc *nsenterFileTypeChecker) IsDir() bool { + return ftc.checkMimetype("directory") +} + +func (ftc *nsenterFileTypeChecker) MakeDir() error { + args := []string{ + fmt.Sprintf("--mount=%s", hostProcMountsNamespace), + "--", + mkdirCmd, + "-p", + ftc.path, + } + exec := exec.New() + if _, err := exec.Command(nsenterCmd, args...).CombinedOutput(); err != nil { + return err + } + return nil +} + +func (ftc *nsenterFileTypeChecker) IsBlock() bool { + return ftc.checkMimetype("block special file") +} + +func (ftc *nsenterFileTypeChecker) IsChar() bool { + return ftc.checkMimetype("character special file") +} + +func (ftc *nsenterFileTypeChecker) IsSocket() bool { + return ftc.checkMimetype("socket") +} + +func (ftc *nsenterFileTypeChecker) GetPath() string { + return ftc.path +} + +func (ftc *nsenterFileTypeChecker) checkMimetype(checkedType string) bool { + if !ftc.Exists() { + return false + } + + args := []string{ + fmt.Sprintf("--mount=%s", hostProcMountsNamespace), + "--", + statCmd, + "-L", + `--printf "%F"`, + ftc.path, + } + exec := exec.New() + outputBytes, err := exec.Command(nsenterCmd, args...).CombinedOutput() + if err != nil { + return false + } + return string(outputBytes) == checkedType +} diff --git a/pkg/volume/host_path/nsenter_unsupported.go b/pkg/volume/host_path/nsenter_unsupported.go new file mode 100644 index 00000000000..166ef9f0a39 --- /dev/null +++ b/pkg/volume/host_path/nsenter_unsupported.go @@ -0,0 +1,66 @@ +// +build !linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package host_path + +type nsenterFileTypeChecker struct { + path string + exists bool +} + +func newNsenterFileTypeChecker(path string) (hostPathTypeChecker, error) { + ftc := &nsenterFileTypeChecker{path: path} + ftc.Exists() + return ftc, nil +} + +func (ftc *nsenterFileTypeChecker) Exists() bool { + return false +} + +func (ftc *nsenterFileTypeChecker) IsFile() bool { + return false +} + +func (ftc *nsenterFileTypeChecker) MakeFile() error { + return nil +} + +func (ftc *nsenterFileTypeChecker) IsDir() bool { + return false +} + +func (ftc *nsenterFileTypeChecker) MakeDir() error { + return nil +} + +func (ftc *nsenterFileTypeChecker) IsBlock() bool { + return false +} + +func (ftc *nsenterFileTypeChecker) IsChar() bool { + return false +} + +func (ftc *nsenterFileTypeChecker) IsSocket() bool { + return false +} + +func (ftc *nsenterFileTypeChecker) GetPath() string { + return ftc.path +}