From 48ed8f3c4a96bb140138f19c03e013ecb84197cb Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Tue, 9 Feb 2021 22:00:53 -0800 Subject: [PATCH] runtime: add support for readonly sandbox bindmounts If specified, sandbox_bind_mounts identifies host paths to be mounted (ro) into the sandboxes shared path. This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted (ro) into the shared fs directory on the host, and thus mapped into the guest. If defaults are utilized, these mounts should be available in the guest at `/var/run/kata-containers/shared/containers/sandbox-mounts` These will not be exposed to the container workloads, and are only added for potential guest-services to consume (example: expose certs into the guest that are available on the host). Fixes: #1464 Signed-off-by: Eric Ernst --- src/runtime/Makefile | 3 + .../cli/config/configuration-clh.toml.in | 6 ++ .../cli/config/configuration-qemu.toml.in | 6 ++ src/runtime/pkg/katautils/config.go | 32 +++++++++++ src/runtime/pkg/katautils/config_test.go | 49 +++++++++++++++++ src/runtime/virtcontainers/kata_agent.go | 55 +++++++++++++++++++ src/runtime/virtcontainers/pkg/oci/utils.go | 4 ++ src/runtime/virtcontainers/sandbox.go | 3 + 8 files changed, 158 insertions(+) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 764c12780c..685b715801 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -217,6 +217,8 @@ DEFPCIEROOTPORT := 0 # Default cgroup model DEFSANDBOXCGROUPONLY ?= false +DEFBINDMOUNTS := [] + # Features FEATURE_SELINUX ?= check @@ -492,6 +494,7 @@ USER_VARS += DEFHOTPLUGVFIOONROOTBUS USER_VARS += DEFPCIEROOTPORT USER_VARS += DEFENTROPYSOURCE USER_VARS += DEFSANDBOXCGROUPONLY +USER_VARS += DEFBINDMOUNTS USER_VARS += FEATURE_SELINUX USER_VARS += BUILDFLAGS diff --git a/src/runtime/cli/config/configuration-clh.toml.in b/src/runtime/cli/config/configuration-clh.toml.in index 50f179e354..72e5fc4be0 100644 --- a/src/runtime/cli/config/configuration-clh.toml.in +++ b/src/runtime/cli/config/configuration-clh.toml.in @@ -252,6 +252,12 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ # See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ +# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path. +# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory. +# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts` +# These will not be exposed to the container workloads, and are only provided for potential guest services. +sandbox_bind_mounts=@DEFBINDMOUNTS@ + # Enabled experimental feature list, format: ["a", "b"]. # Experimental features are features not stable enough for production, # they may break compatibility, and are prepared for a big version bump. diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index 93c96e60b6..49f5e5ae08 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -511,6 +511,12 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ # See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ +# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path. +# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory. +# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts` +# These will not be exposed to the container workloads, and are only provided for potential guest services. +sandbox_bind_mounts=@DEFBINDMOUNTS@ + # Enabled experimental feature list, format: ["a", "b"]. # Experimental features are features not stable enough for production, # they may break compatibility, and are prepared for a big version bump. diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 798e649e23..d10037501f 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io/ioutil" + "path/filepath" goruntime "runtime" "strings" @@ -136,6 +137,7 @@ type runtime struct { DisableNewNetNs bool `toml:"disable_new_netns"` DisableGuestSeccomp bool `toml:"disable_guest_seccomp"` SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` + SandboxBindMounts []string `toml:"sandbox_bind_mounts"` Experimental []string `toml:"experimental"` InterNetworkModel string `toml:"internetworking_model"` EnablePprof bool `toml:"enable_pprof"` @@ -1158,6 +1160,11 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved config.Experimental = append(config.Experimental, *feature) } + if err = validateBindMounts(tomlConf.Runtime.SandboxBindMounts); err != nil { + return "", config, err + } + config.SandboxBindMounts = tomlConf.Runtime.SandboxBindMounts + if err := checkConfig(config); err != nil { return "", config, err } @@ -1165,6 +1172,31 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved return resolved, config, nil } +// Verify that bind mounts exist +func validateBindMounts(mounts []string) error { + if len(mounts) == 0 { + return nil + } + + bases := make(map[string]struct{}) + + for _, m := range mounts { + path, err := ResolvePath(m) + if err != nil { + return fmt.Errorf("sandbox-bindmounts: Failed to resolve path: %s: %v", m, err) + } + + base := filepath.Base(path) + // check to make sure the base does not already exists. + if _, ok := bases[base]; !ok { + bases[base] = struct{}{} + } else { + return fmt.Errorf("sandbox-bindmounts: File %s has base that matches already specified bindmount", path) + } + } + return nil +} + func decodeConfig(configPath string) (tomlConfig, string, error) { var ( resolved string diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index 6d454a73fb..eb6b7d6a39 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -1610,3 +1610,52 @@ func TestCheckFactoryConfig(t *testing.T) { } } } + +func TestValidateBindMounts(t *testing.T) { + assert := assert.New(t) + + tmpdir1, err := ioutil.TempDir(testDir, "tmp1-") + assert.NoError(err) + defer os.RemoveAll(tmpdir1) + + tmpdir2, err := ioutil.TempDir(testDir, "tmp2-") + assert.NoError(err) + defer os.RemoveAll(tmpdir2) + + duplicate1 := filepath.Join(tmpdir1, "cat.txt") + duplicate2 := filepath.Join(tmpdir2, "cat.txt") + unique := filepath.Join(tmpdir1, "foobar.txt") + + err = ioutil.WriteFile(duplicate1, []byte("kibble-monster"), 0644) + assert.NoError(err) + + err = ioutil.WriteFile(duplicate2, []byte("furbag"), 0644) + assert.NoError(err) + + err = ioutil.WriteFile(unique, []byte("fuzzball"), 0644) + assert.NoError(err) + + type testData struct { + name string + mounts []string + expectError bool + } + + data := []testData{ + {"two unique directories", []string{tmpdir1, tmpdir2}, false}, + {"unique directory and two unique files", []string{tmpdir1, duplicate1, unique}, false}, + {"two files with same base name", []string{duplicate1, duplicate2}, true}, + {"non existent path", []string{"/this/does/not/exist"}, true}, + {"non existent path with existing path", []string{unique, "/this/does/not/exist"}, true}, + {"non existent path with duplicates", []string{duplicate1, duplicate2, "/this/does/not/exist"}, true}, + {"no paths", []string{}, false}, + } + for i, d := range data { + err := validateBindMounts(d.mounts) + if d.expectError { + assert.Error(err, "test %d (%+v)", i, d.name) + } else { + assert.NoError(err, "test %d (%+v)", i, d.name) + } + } +} diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 9492a2f731..46140ee192 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -55,6 +55,8 @@ const ( // path to vfio devices vfioPath = "/dev/vfio/" + sandboxMountsDir = "sandbox-mounts" + // enable debug console kernelParamDebugConsole = "agent.debug_console" kernelParamDebugConsoleVPort = "agent.debug_console_vport" @@ -379,6 +381,50 @@ func (k *kataAgent) internalConfigure(h hypervisor, id string, config interface{ return nil } +func setupSandboxBindMounts(sandbox *Sandbox) error { + if len(sandbox.config.SandboxBindMounts) == 0 { + return nil + } + + // Create subdirectory in host shared path for sandbox mounts + sandboxMountDir := filepath.Join(getMountPath(sandbox.id), sandboxMountsDir) + sandboxShareDir := filepath.Join(getSharePath(sandbox.id), sandboxMountsDir) + if err := os.MkdirAll(sandboxMountDir, DirMode); err != nil { + return fmt.Errorf("Creating sandbox shared mount directory: %v: %w", sandboxMountDir, err) + } + + for _, m := range sandbox.config.SandboxBindMounts { + mountDest := filepath.Join(sandboxMountDir, filepath.Base(m)) + // bind-mount each sandbox mount that's defined into the sandbox mounts dir + if err := bindMount(context.Background(), m, mountDest, true, "private"); err != nil { + return fmt.Errorf("Mounting sandbox directory: %v to %v: %w", m, mountDest, err) + } + + mountDest = filepath.Join(sandboxShareDir, filepath.Base(m)) + if err := remountRo(context.Background(), mountDest); err != nil { + return fmt.Errorf("remount sandbox directory: %v to %v: %w", m, mountDest, err) + } + + } + + return nil +} + +func cleanupSandboxBindMounts(sandbox *Sandbox) error { + if len(sandbox.config.SandboxBindMounts) == 0 { + return nil + } + + for _, m := range sandbox.config.SandboxBindMounts { + mountPath := filepath.Join(getMountPath(sandbox.id), sandboxMountsDir, filepath.Base(m)) + if err := syscall.Unmount(mountPath, syscall.MNT_DETACH|UmountNoFollow); err != nil { + return fmt.Errorf("Unmounting observe directory: %v: %w", mountPath, err) + } + } + + return nil +} + func (k *kataAgent) configure(h hypervisor, id, sharePath string, config interface{}) error { err := k.internalConfigure(h, id, config) if err != nil { @@ -441,6 +487,11 @@ func (k *kataAgent) setupSharedPath(sandbox *Sandbox) error { return err } + // Setup sandbox bindmounts, if specified: + if err := setupSandboxBindMounts(sandbox); err != nil { + return err + } + return nil } @@ -2099,6 +2150,10 @@ func (k *kataAgent) markDead() { } func (k *kataAgent) cleanup(s *Sandbox) { + if err := cleanupSandboxBindMounts(s); err != nil { + k.Logger().WithError(err).Errorf("failed to cleanup observability logs bindmount") + } + // Unmount shared path path := getSharePath(s.id) k.Logger().WithField("path", path).Infof("cleanup agent") diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index 6310d8e235..1fa885bb80 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -122,6 +122,9 @@ type RuntimeConfig struct { //Determines kata processes are managed only in sandbox cgroup SandboxCgroupOnly bool + //Paths to be bindmounted RO into the guest. + SandboxBindMounts []string + //Experimental features enabled Experimental []exp.Feature @@ -964,6 +967,7 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c SystemdCgroup: systemdCgroup, SandboxCgroupOnly: runtime.SandboxCgroupOnly, + SandboxBindMounts: runtime.SandboxBindMounts, DisableGuestSeccomp: runtime.DisableGuestSeccomp, diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index a2177a4f75..42811852fb 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -116,6 +116,9 @@ type SandboxConfig struct { DisableGuestSeccomp bool + // SandboxBindMounts - list of paths to mount into guest + SandboxBindMounts []string + // Experimental features enabled Experimental []exp.Feature