diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 764c12780..685b71580 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -217,6 +217,8 @@ DEFPCIEROOTPORT := 0 # Default cgroup model DEFSANDBOXCGROUPONLY ?= false +DEFBINDMOUNTS := [] + # Features FEATURE_SELINUX ?= check @@ -492,6 +494,7 @@ USER_VARS += DEFHOTPLUGVFIOONROOTBUS USER_VARS += DEFPCIEROOTPORT USER_VARS += DEFENTROPYSOURCE USER_VARS += DEFSANDBOXCGROUPONLY +USER_VARS += DEFBINDMOUNTS USER_VARS += FEATURE_SELINUX USER_VARS += BUILDFLAGS diff --git a/src/runtime/cli/config/configuration-clh.toml.in b/src/runtime/cli/config/configuration-clh.toml.in index 50f179e35..72e5fc4be 100644 --- a/src/runtime/cli/config/configuration-clh.toml.in +++ b/src/runtime/cli/config/configuration-clh.toml.in @@ -252,6 +252,12 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ # See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ +# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path. +# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory. +# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts` +# These will not be exposed to the container workloads, and are only provided for potential guest services. +sandbox_bind_mounts=@DEFBINDMOUNTS@ + # Enabled experimental feature list, format: ["a", "b"]. # Experimental features are features not stable enough for production, # they may break compatibility, and are prepared for a big version bump. diff --git a/src/runtime/cli/config/configuration-qemu.toml.in b/src/runtime/cli/config/configuration-qemu.toml.in index 93c96e60b..49f5e5ae0 100644 --- a/src/runtime/cli/config/configuration-qemu.toml.in +++ b/src/runtime/cli/config/configuration-qemu.toml.in @@ -511,6 +511,12 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ # See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ +# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path. +# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory. +# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts` +# These will not be exposed to the container workloads, and are only provided for potential guest services. +sandbox_bind_mounts=@DEFBINDMOUNTS@ + # Enabled experimental feature list, format: ["a", "b"]. # Experimental features are features not stable enough for production, # they may break compatibility, and are prepared for a big version bump. diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 798e649e2..d10037501 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io/ioutil" + "path/filepath" goruntime "runtime" "strings" @@ -136,6 +137,7 @@ type runtime struct { DisableNewNetNs bool `toml:"disable_new_netns"` DisableGuestSeccomp bool `toml:"disable_guest_seccomp"` SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` + SandboxBindMounts []string `toml:"sandbox_bind_mounts"` Experimental []string `toml:"experimental"` InterNetworkModel string `toml:"internetworking_model"` EnablePprof bool `toml:"enable_pprof"` @@ -1158,6 +1160,11 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved config.Experimental = append(config.Experimental, *feature) } + if err = validateBindMounts(tomlConf.Runtime.SandboxBindMounts); err != nil { + return "", config, err + } + config.SandboxBindMounts = tomlConf.Runtime.SandboxBindMounts + if err := checkConfig(config); err != nil { return "", config, err } @@ -1165,6 +1172,31 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved return resolved, config, nil } +// Verify that bind mounts exist +func validateBindMounts(mounts []string) error { + if len(mounts) == 0 { + return nil + } + + bases := make(map[string]struct{}) + + for _, m := range mounts { + path, err := ResolvePath(m) + if err != nil { + return fmt.Errorf("sandbox-bindmounts: Failed to resolve path: %s: %v", m, err) + } + + base := filepath.Base(path) + // check to make sure the base does not already exists. + if _, ok := bases[base]; !ok { + bases[base] = struct{}{} + } else { + return fmt.Errorf("sandbox-bindmounts: File %s has base that matches already specified bindmount", path) + } + } + return nil +} + func decodeConfig(configPath string) (tomlConfig, string, error) { var ( resolved string diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index 6d454a73f..eb6b7d6a3 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -1610,3 +1610,52 @@ func TestCheckFactoryConfig(t *testing.T) { } } } + +func TestValidateBindMounts(t *testing.T) { + assert := assert.New(t) + + tmpdir1, err := ioutil.TempDir(testDir, "tmp1-") + assert.NoError(err) + defer os.RemoveAll(tmpdir1) + + tmpdir2, err := ioutil.TempDir(testDir, "tmp2-") + assert.NoError(err) + defer os.RemoveAll(tmpdir2) + + duplicate1 := filepath.Join(tmpdir1, "cat.txt") + duplicate2 := filepath.Join(tmpdir2, "cat.txt") + unique := filepath.Join(tmpdir1, "foobar.txt") + + err = ioutil.WriteFile(duplicate1, []byte("kibble-monster"), 0644) + assert.NoError(err) + + err = ioutil.WriteFile(duplicate2, []byte("furbag"), 0644) + assert.NoError(err) + + err = ioutil.WriteFile(unique, []byte("fuzzball"), 0644) + assert.NoError(err) + + type testData struct { + name string + mounts []string + expectError bool + } + + data := []testData{ + {"two unique directories", []string{tmpdir1, tmpdir2}, false}, + {"unique directory and two unique files", []string{tmpdir1, duplicate1, unique}, false}, + {"two files with same base name", []string{duplicate1, duplicate2}, true}, + {"non existent path", []string{"/this/does/not/exist"}, true}, + {"non existent path with existing path", []string{unique, "/this/does/not/exist"}, true}, + {"non existent path with duplicates", []string{duplicate1, duplicate2, "/this/does/not/exist"}, true}, + {"no paths", []string{}, false}, + } + for i, d := range data { + err := validateBindMounts(d.mounts) + if d.expectError { + assert.Error(err, "test %d (%+v)", i, d.name) + } else { + assert.NoError(err, "test %d (%+v)", i, d.name) + } + } +} diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 9492a2f73..46140ee19 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -55,6 +55,8 @@ const ( // path to vfio devices vfioPath = "/dev/vfio/" + sandboxMountsDir = "sandbox-mounts" + // enable debug console kernelParamDebugConsole = "agent.debug_console" kernelParamDebugConsoleVPort = "agent.debug_console_vport" @@ -379,6 +381,50 @@ func (k *kataAgent) internalConfigure(h hypervisor, id string, config interface{ return nil } +func setupSandboxBindMounts(sandbox *Sandbox) error { + if len(sandbox.config.SandboxBindMounts) == 0 { + return nil + } + + // Create subdirectory in host shared path for sandbox mounts + sandboxMountDir := filepath.Join(getMountPath(sandbox.id), sandboxMountsDir) + sandboxShareDir := filepath.Join(getSharePath(sandbox.id), sandboxMountsDir) + if err := os.MkdirAll(sandboxMountDir, DirMode); err != nil { + return fmt.Errorf("Creating sandbox shared mount directory: %v: %w", sandboxMountDir, err) + } + + for _, m := range sandbox.config.SandboxBindMounts { + mountDest := filepath.Join(sandboxMountDir, filepath.Base(m)) + // bind-mount each sandbox mount that's defined into the sandbox mounts dir + if err := bindMount(context.Background(), m, mountDest, true, "private"); err != nil { + return fmt.Errorf("Mounting sandbox directory: %v to %v: %w", m, mountDest, err) + } + + mountDest = filepath.Join(sandboxShareDir, filepath.Base(m)) + if err := remountRo(context.Background(), mountDest); err != nil { + return fmt.Errorf("remount sandbox directory: %v to %v: %w", m, mountDest, err) + } + + } + + return nil +} + +func cleanupSandboxBindMounts(sandbox *Sandbox) error { + if len(sandbox.config.SandboxBindMounts) == 0 { + return nil + } + + for _, m := range sandbox.config.SandboxBindMounts { + mountPath := filepath.Join(getMountPath(sandbox.id), sandboxMountsDir, filepath.Base(m)) + if err := syscall.Unmount(mountPath, syscall.MNT_DETACH|UmountNoFollow); err != nil { + return fmt.Errorf("Unmounting observe directory: %v: %w", mountPath, err) + } + } + + return nil +} + func (k *kataAgent) configure(h hypervisor, id, sharePath string, config interface{}) error { err := k.internalConfigure(h, id, config) if err != nil { @@ -441,6 +487,11 @@ func (k *kataAgent) setupSharedPath(sandbox *Sandbox) error { return err } + // Setup sandbox bindmounts, if specified: + if err := setupSandboxBindMounts(sandbox); err != nil { + return err + } + return nil } @@ -2099,6 +2150,10 @@ func (k *kataAgent) markDead() { } func (k *kataAgent) cleanup(s *Sandbox) { + if err := cleanupSandboxBindMounts(s); err != nil { + k.Logger().WithError(err).Errorf("failed to cleanup observability logs bindmount") + } + // Unmount shared path path := getSharePath(s.id) k.Logger().WithField("path", path).Infof("cleanup agent") diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index 6310d8e23..1fa885bb8 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -122,6 +122,9 @@ type RuntimeConfig struct { //Determines kata processes are managed only in sandbox cgroup SandboxCgroupOnly bool + //Paths to be bindmounted RO into the guest. + SandboxBindMounts []string + //Experimental features enabled Experimental []exp.Feature @@ -964,6 +967,7 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c SystemdCgroup: systemdCgroup, SandboxCgroupOnly: runtime.SandboxCgroupOnly, + SandboxBindMounts: runtime.SandboxBindMounts, DisableGuestSeccomp: runtime.DisableGuestSeccomp, diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index a2177a4f7..42811852f 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -116,6 +116,9 @@ type SandboxConfig struct { DisableGuestSeccomp bool + // SandboxBindMounts - list of paths to mount into guest + SandboxBindMounts []string + // Experimental features enabled Experimental []exp.Feature