runtime: add support for readonly sandbox bindmounts

If specified, sandbox_bind_mounts identifies host paths to be
mounted (ro) into the sandboxes shared path. This is only valid
if filesystem sharing is utilized.

The provided path(s) will be bindmounted (ro) into the shared fs directory on
the host, and thus mapped into the guest. If defaults are utilized,
these mounts should be available in the guest at
`/var/run/kata-containers/shared/containers/sandbox-mounts`

These will not be exposed to the container workloads, and are only
added for potential guest-services to consume (example: expose certs
into the guest that are available on the host).

Fixes: #1464

Signed-off-by: Eric Ernst <eric.g.ernst@gmail.com>

src/runtime/Makefile

@@ -217,6 +217,8 @@ DEFPCIEROOTPORT := 0
 # Default cgroup model
 DEFSANDBOXCGROUPONLY ?= false
 
+DEFBINDMOUNTS := []
+
 # Features
 FEATURE_SELINUX ?= check
 
@@ -492,6 +494,7 @@ USER_VARS += DEFHOTPLUGVFIOONROOTBUS
 USER_VARS += DEFPCIEROOTPORT
 USER_VARS += DEFENTROPYSOURCE
 USER_VARS += DEFSANDBOXCGROUPONLY
+USER_VARS += DEFBINDMOUNTS
 USER_VARS += FEATURE_SELINUX
 USER_VARS += BUILDFLAGS
 

src/runtime/cli/config/configuration-clh.toml.in

@@ -252,6 +252,12 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 # See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
 sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 
+# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
+# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
+# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
+# These will not be exposed to the container workloads, and are only provided for potential guest services.
+sandbox_bind_mounts=@DEFBINDMOUNTS@
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/cli/config/configuration-qemu.toml.in

@@ -511,6 +511,12 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 # See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
 sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 
+# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
+# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
+# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
+# These will not be exposed to the container workloads, and are only provided for potential guest services.
+sandbox_bind_mounts=@DEFBINDMOUNTS@
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/pkg/katautils/config.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"path/filepath"
 	goruntime "runtime"
 	"strings"
 
@@ -136,6 +137,7 @@ type runtime struct {
 	DisableNewNetNs     bool     `toml:"disable_new_netns"`
 	DisableGuestSeccomp bool     `toml:"disable_guest_seccomp"`
 	SandboxCgroupOnly   bool     `toml:"sandbox_cgroup_only"`
+	SandboxBindMounts   []string `toml:"sandbox_bind_mounts"`
 	Experimental        []string `toml:"experimental"`
 	InterNetworkModel   string   `toml:"internetworking_model"`
 	EnablePprof         bool     `toml:"enable_pprof"`
@@ -1158,6 +1160,11 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
 		config.Experimental = append(config.Experimental, *feature)
 	}
 
+	if err = validateBindMounts(tomlConf.Runtime.SandboxBindMounts); err != nil {
+		return "", config, err
+	}
+	config.SandboxBindMounts = tomlConf.Runtime.SandboxBindMounts
+
 	if err := checkConfig(config); err != nil {
 		return "", config, err
 	}
@@ -1165,6 +1172,31 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
 	return resolved, config, nil
 }
 
+// Verify that bind mounts exist
+func validateBindMounts(mounts []string) error {
+	if len(mounts) == 0 {
+		return nil
+	}
+
+	bases := make(map[string]struct{})
+
+	for _, m := range mounts {
+		path, err := ResolvePath(m)
+		if err != nil {
+			return fmt.Errorf("sandbox-bindmounts: Failed to resolve path: %s: %v", m, err)
+		}
+
+		base := filepath.Base(path)
+		// check to make sure the base does not already exists.
+		if _, ok := bases[base]; !ok {
+			bases[base] = struct{}{}
+		} else {
+			return fmt.Errorf("sandbox-bindmounts: File %s has base that matches already specified bindmount", path)
+		}
+	}
+	return nil
+}
+
 func decodeConfig(configPath string) (tomlConfig, string, error) {
 	var (
 		resolved string

src/runtime/pkg/katautils/config_test.go

@@ -1610,3 +1610,52 @@ func TestCheckFactoryConfig(t *testing.T) {
 		}
 	}
 }
+
+func TestValidateBindMounts(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir1, err := ioutil.TempDir(testDir, "tmp1-")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir1)
+
+	tmpdir2, err := ioutil.TempDir(testDir, "tmp2-")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir2)
+
+	duplicate1 := filepath.Join(tmpdir1, "cat.txt")
+	duplicate2 := filepath.Join(tmpdir2, "cat.txt")
+	unique := filepath.Join(tmpdir1, "foobar.txt")
+
+	err = ioutil.WriteFile(duplicate1, []byte("kibble-monster"), 0644)
+	assert.NoError(err)
+
+	err = ioutil.WriteFile(duplicate2, []byte("furbag"), 0644)
+	assert.NoError(err)
+
+	err = ioutil.WriteFile(unique, []byte("fuzzball"), 0644)
+	assert.NoError(err)
+
+	type testData struct {
+		name        string
+		mounts      []string
+		expectError bool
+	}
+
+	data := []testData{
+		{"two unique directories", []string{tmpdir1, tmpdir2}, false},
+		{"unique directory and two unique files", []string{tmpdir1, duplicate1, unique}, false},
+		{"two files with same base name", []string{duplicate1, duplicate2}, true},
+		{"non existent path", []string{"/this/does/not/exist"}, true},
+		{"non existent path with existing path", []string{unique, "/this/does/not/exist"}, true},
+		{"non existent path with duplicates", []string{duplicate1, duplicate2, "/this/does/not/exist"}, true},
+		{"no paths", []string{}, false},
+	}
+	for i, d := range data {
+		err := validateBindMounts(d.mounts)
+		if d.expectError {
+			assert.Error(err, "test %d (%+v)", i, d.name)
+		} else {
+			assert.NoError(err, "test %d (%+v)", i, d.name)
+		}
+	}
+}

src/runtime/virtcontainers/kata_agent.go

@@ -55,6 +55,8 @@ const (
 	// path to vfio devices
 	vfioPath = "/dev/vfio/"
 
+	sandboxMountsDir = "sandbox-mounts"
+
 	// enable debug console
 	kernelParamDebugConsole           = "agent.debug_console"
 	kernelParamDebugConsoleVPort      = "agent.debug_console_vport"
@@ -379,6 +381,50 @@ func (k *kataAgent) internalConfigure(h hypervisor, id string, config interface{
 	return nil
 }
 
+func setupSandboxBindMounts(sandbox *Sandbox) error {
+	if len(sandbox.config.SandboxBindMounts) == 0 {
+		return nil
+	}
+
+	// Create subdirectory in host shared path for sandbox mounts
+	sandboxMountDir := filepath.Join(getMountPath(sandbox.id), sandboxMountsDir)
+	sandboxShareDir := filepath.Join(getSharePath(sandbox.id), sandboxMountsDir)
+	if err := os.MkdirAll(sandboxMountDir, DirMode); err != nil {
+		return fmt.Errorf("Creating sandbox shared mount directory: %v: %w", sandboxMountDir, err)
+	}
+
+	for _, m := range sandbox.config.SandboxBindMounts {
+		mountDest := filepath.Join(sandboxMountDir, filepath.Base(m))
+		// bind-mount each sandbox mount that's defined into the sandbox mounts dir
+		if err := bindMount(context.Background(), m, mountDest, true, "private"); err != nil {
+			return fmt.Errorf("Mounting sandbox directory: %v to %v: %w", m, mountDest, err)
+		}
+
+		mountDest = filepath.Join(sandboxShareDir, filepath.Base(m))
+		if err := remountRo(context.Background(), mountDest); err != nil {
+			return fmt.Errorf("remount sandbox directory: %v to %v: %w", m, mountDest, err)
+		}
+
+	}
+
+	return nil
+}
+
+func cleanupSandboxBindMounts(sandbox *Sandbox) error {
+	if len(sandbox.config.SandboxBindMounts) == 0 {
+		return nil
+	}
+
+	for _, m := range sandbox.config.SandboxBindMounts {
+		mountPath := filepath.Join(getMountPath(sandbox.id), sandboxMountsDir, filepath.Base(m))
+		if err := syscall.Unmount(mountPath, syscall.MNT_DETACH|UmountNoFollow); err != nil {
+			return fmt.Errorf("Unmounting observe directory: %v: %w", mountPath, err)
+		}
+	}
+
+	return nil
+}
+
 func (k *kataAgent) configure(h hypervisor, id, sharePath string, config interface{}) error {
 	err := k.internalConfigure(h, id, config)
 	if err != nil {
@@ -441,6 +487,11 @@ func (k *kataAgent) setupSharedPath(sandbox *Sandbox) error {
 		return err
 	}
 
+	// Setup sandbox bindmounts, if specified:
+	if err := setupSandboxBindMounts(sandbox); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -2099,6 +2150,10 @@ func (k *kataAgent) markDead() {
 }
 
 func (k *kataAgent) cleanup(s *Sandbox) {
+	if err := cleanupSandboxBindMounts(s); err != nil {
+		k.Logger().WithError(err).Errorf("failed to cleanup observability logs bindmount")
+	}
+
 	// Unmount shared path
 	path := getSharePath(s.id)
 	k.Logger().WithField("path", path).Infof("cleanup agent")

src/runtime/virtcontainers/pkg/oci/utils.go

@@ -122,6 +122,9 @@ type RuntimeConfig struct {
 	//Determines kata processes are managed only in sandbox cgroup
 	SandboxCgroupOnly bool
 
+	//Paths to be bindmounted RO into the guest.
+	SandboxBindMounts []string
+
 	//Experimental features enabled
 	Experimental []exp.Feature
 
@@ -964,6 +967,7 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c
 		SystemdCgroup: systemdCgroup,
 
 		SandboxCgroupOnly: runtime.SandboxCgroupOnly,
+		SandboxBindMounts: runtime.SandboxBindMounts,
 
 		DisableGuestSeccomp: runtime.DisableGuestSeccomp,
 

src/runtime/virtcontainers/sandbox.go

@@ -116,6 +116,9 @@ type SandboxConfig struct {
 
 	DisableGuestSeccomp bool
 
+	// SandboxBindMounts - list of paths to mount into guest
+	SandboxBindMounts []string
+
 	// Experimental features enabled
 	Experimental []exp.Feature