Merge pull request #1463 from egernst/bindmount-infra

runtime: add support for readonly sandbox bindmounts

src/runtime/Makefile

@@ -217,6 +217,8 @@ DEFPCIEROOTPORT := 0
 # Default cgroup model
 DEFSANDBOXCGROUPONLY ?= false
 
+DEFBINDMOUNTS := []
+
 # Features
 FEATURE_SELINUX ?= check
 
@@ -492,6 +494,7 @@ USER_VARS += DEFHOTPLUGVFIOONROOTBUS
 USER_VARS += DEFPCIEROOTPORT
 USER_VARS += DEFENTROPYSOURCE
 USER_VARS += DEFSANDBOXCGROUPONLY
+USER_VARS += DEFBINDMOUNTS
 USER_VARS += FEATURE_SELINUX
 USER_VARS += BUILDFLAGS
 

src/runtime/cli/config/configuration-clh.toml.in

@@ -252,6 +252,12 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 # See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
 sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 
+# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
+# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
+# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
+# These will not be exposed to the container workloads, and are only provided for potential guest services.
+sandbox_bind_mounts=@DEFBINDMOUNTS@
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/cli/config/configuration-qemu.toml.in

@@ -511,6 +511,12 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 # See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
 sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@
 
+# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path.
+# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory.
+# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts`
+# These will not be exposed to the container workloads, and are only provided for potential guest services.
+sandbox_bind_mounts=@DEFBINDMOUNTS@
+
 # Enabled experimental feature list, format: ["a", "b"].
 # Experimental features are features not stable enough for production,
 # they may break compatibility, and are prepared for a big version bump.

src/runtime/pkg/katautils/config.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"path/filepath"
 	goruntime "runtime"
 	"strings"
 
@@ -136,6 +137,7 @@ type runtime struct {
 	DisableNewNetNs     bool     `toml:"disable_new_netns"`
 	DisableGuestSeccomp bool     `toml:"disable_guest_seccomp"`
 	SandboxCgroupOnly   bool     `toml:"sandbox_cgroup_only"`
+	SandboxBindMounts   []string `toml:"sandbox_bind_mounts"`
 	Experimental        []string `toml:"experimental"`
 	InterNetworkModel   string   `toml:"internetworking_model"`
 	EnablePprof         bool     `toml:"enable_pprof"`
@@ -1158,6 +1160,11 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
 		config.Experimental = append(config.Experimental, *feature)
 	}
 
+	if err = validateBindMounts(tomlConf.Runtime.SandboxBindMounts); err != nil {
+		return "", config, err
+	}
+	config.SandboxBindMounts = tomlConf.Runtime.SandboxBindMounts
+
 	if err := checkConfig(config); err != nil {
 		return "", config, err
 	}
@@ -1165,6 +1172,31 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved
 	return resolved, config, nil
 }
 
+// Verify that bind mounts exist
+func validateBindMounts(mounts []string) error {
+	if len(mounts) == 0 {
+		return nil
+	}
+
+	bases := make(map[string]struct{})
+
+	for _, m := range mounts {
+		path, err := ResolvePath(m)
+		if err != nil {
+			return fmt.Errorf("sandbox-bindmounts: Failed to resolve path: %s: %v", m, err)
+		}
+
+		base := filepath.Base(path)
+		// check to make sure the base does not already exists.
+		if _, ok := bases[base]; !ok {
+			bases[base] = struct{}{}
+		} else {
+			return fmt.Errorf("sandbox-bindmounts: File %s has base that matches already specified bindmount", path)
+		}
+	}
+	return nil
+}
+
 func decodeConfig(configPath string) (tomlConfig, string, error) {
 	var (
 		resolved string

src/runtime/pkg/katautils/config_test.go

@@ -1610,3 +1610,52 @@ func TestCheckFactoryConfig(t *testing.T) {
 		}
 	}
 }
+
+func TestValidateBindMounts(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpdir1, err := ioutil.TempDir(testDir, "tmp1-")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir1)
+
+	tmpdir2, err := ioutil.TempDir(testDir, "tmp2-")
+	assert.NoError(err)
+	defer os.RemoveAll(tmpdir2)
+
+	duplicate1 := filepath.Join(tmpdir1, "cat.txt")
+	duplicate2 := filepath.Join(tmpdir2, "cat.txt")
+	unique := filepath.Join(tmpdir1, "foobar.txt")
+
+	err = ioutil.WriteFile(duplicate1, []byte("kibble-monster"), 0644)
+	assert.NoError(err)
+
+	err = ioutil.WriteFile(duplicate2, []byte("furbag"), 0644)
+	assert.NoError(err)
+
+	err = ioutil.WriteFile(unique, []byte("fuzzball"), 0644)
+	assert.NoError(err)
+
+	type testData struct {
+		name        string
+		mounts      []string
+		expectError bool
+	}
+
+	data := []testData{
+		{"two unique directories", []string{tmpdir1, tmpdir2}, false},
+		{"unique directory and two unique files", []string{tmpdir1, duplicate1, unique}, false},
+		{"two files with same base name", []string{duplicate1, duplicate2}, true},
+		{"non existent path", []string{"/this/does/not/exist"}, true},
+		{"non existent path with existing path", []string{unique, "/this/does/not/exist"}, true},
+		{"non existent path with duplicates", []string{duplicate1, duplicate2, "/this/does/not/exist"}, true},
+		{"no paths", []string{}, false},
+	}
+	for i, d := range data {
+		err := validateBindMounts(d.mounts)
+		if d.expectError {
+			assert.Error(err, "test %d (%+v)", i, d.name)
+		} else {
+			assert.NoError(err, "test %d (%+v)", i, d.name)
+		}
+	}
+}

src/runtime/virtcontainers/kata_agent.go

@@ -55,6 +55,8 @@ const (
 	// path to vfio devices
 	vfioPath = "/dev/vfio/"
 
+	sandboxMountsDir = "sandbox-mounts"
+
 	// enable debug console
 	kernelParamDebugConsole           = "agent.debug_console"
 	kernelParamDebugConsoleVPort      = "agent.debug_console_vport"
@@ -379,6 +381,50 @@ func (k *kataAgent) internalConfigure(h hypervisor, id string, config interface{
 	return nil
 }
 
+func setupSandboxBindMounts(sandbox *Sandbox) error {
+	if len(sandbox.config.SandboxBindMounts) == 0 {
+		return nil
+	}
+
+	// Create subdirectory in host shared path for sandbox mounts
+	sandboxMountDir := filepath.Join(getMountPath(sandbox.id), sandboxMountsDir)
+	sandboxShareDir := filepath.Join(getSharePath(sandbox.id), sandboxMountsDir)
+	if err := os.MkdirAll(sandboxMountDir, DirMode); err != nil {
+		return fmt.Errorf("Creating sandbox shared mount directory: %v: %w", sandboxMountDir, err)
+	}
+
+	for _, m := range sandbox.config.SandboxBindMounts {
+		mountDest := filepath.Join(sandboxMountDir, filepath.Base(m))
+		// bind-mount each sandbox mount that's defined into the sandbox mounts dir
+		if err := bindMount(context.Background(), m, mountDest, true, "private"); err != nil {
+			return fmt.Errorf("Mounting sandbox directory: %v to %v: %w", m, mountDest, err)
+		}
+
+		mountDest = filepath.Join(sandboxShareDir, filepath.Base(m))
+		if err := remountRo(context.Background(), mountDest); err != nil {
+			return fmt.Errorf("remount sandbox directory: %v to %v: %w", m, mountDest, err)
+		}
+
+	}
+
+	return nil
+}
+
+func cleanupSandboxBindMounts(sandbox *Sandbox) error {
+	if len(sandbox.config.SandboxBindMounts) == 0 {
+		return nil
+	}
+
+	for _, m := range sandbox.config.SandboxBindMounts {
+		mountPath := filepath.Join(getMountPath(sandbox.id), sandboxMountsDir, filepath.Base(m))
+		if err := syscall.Unmount(mountPath, syscall.MNT_DETACH|UmountNoFollow); err != nil {
+			return fmt.Errorf("Unmounting observe directory: %v: %w", mountPath, err)
+		}
+	}
+
+	return nil
+}
+
 func (k *kataAgent) configure(h hypervisor, id, sharePath string, config interface{}) error {
 	err := k.internalConfigure(h, id, config)
 	if err != nil {
@@ -441,6 +487,11 @@ func (k *kataAgent) setupSharedPath(sandbox *Sandbox) error {
 		return err
 	}
 
+	// Setup sandbox bindmounts, if specified:
+	if err := setupSandboxBindMounts(sandbox); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -2099,6 +2150,10 @@ func (k *kataAgent) markDead() {
 }
 
 func (k *kataAgent) cleanup(s *Sandbox) {
+	if err := cleanupSandboxBindMounts(s); err != nil {
+		k.Logger().WithError(err).Errorf("failed to cleanup observability logs bindmount")
+	}
+
 	// Unmount shared path
 	path := getSharePath(s.id)
 	k.Logger().WithField("path", path).Infof("cleanup agent")

src/runtime/virtcontainers/pkg/oci/utils.go

@@ -122,6 +122,9 @@ type RuntimeConfig struct {
 	//Determines kata processes are managed only in sandbox cgroup
 	SandboxCgroupOnly bool
 
+	//Paths to be bindmounted RO into the guest.
+	SandboxBindMounts []string
+
 	//Experimental features enabled
 	Experimental []exp.Feature
 
@@ -964,6 +967,7 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c
 		SystemdCgroup: systemdCgroup,
 
 		SandboxCgroupOnly: runtime.SandboxCgroupOnly,
+		SandboxBindMounts: runtime.SandboxBindMounts,
 
 		DisableGuestSeccomp: runtime.DisableGuestSeccomp,
 

src/runtime/virtcontainers/sandbox.go

@@ -116,6 +116,9 @@ type SandboxConfig struct {
 
 	DisableGuestSeccomp bool
 
+	// SandboxBindMounts - list of paths to mount into guest
+	SandboxBindMounts []string
+
 	// Experimental features enabled
 	Experimental []exp.Feature