From 6a21e20c6397f84bdfd393a28a4339c92f0812bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 28 Jun 2023 16:43:08 +0200 Subject: [PATCH] runtime: Add "none" as a shared_fs option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, even when using devmapper, if the VMM supports virtio-fs / virtio-9p, that's used to share a few files between the host and the guest. This *needed*, as we need to share with the guest contents like secrets, certificates, and configurations, via Kubernetes objects like configMaps or secrets, and those are rotated and must be updated into the guest whenever the rotation happens. However, there are still use-cases users can live with just copying those files into the guest at the pod creation time, and for those there's absolutely no need to have a shared filesystem process running with no extra obvious benefit, consuming memory and even increasing the attack surface used by Kata Containers. For the case mentioned above, we should allow users, making it very clear which limitations it'll bring, to run Kata Containers with devmapper without actually having to use a shared file system, which is already the approach taken when using Firecracker as the VMM. Fixes: #7207 Signed-off-by: Fabiano Fidêncio --- src/runtime/config/configuration-clh.toml.in | 5 ++++ .../configuration-qemu-nvidia-gpu.toml.in | 5 ++++ .../config/configuration-qemu-sev.toml.in | 7 ++++- .../config/configuration-qemu-snp.toml.in | 5 ++++ .../config/configuration-qemu-tdx.toml.in | 5 ++++ src/runtime/config/configuration-qemu.toml.in | 5 ++++ src/runtime/pkg/device/config/config.go | 11 ++++++++ src/runtime/pkg/katautils/config.go | 9 ++++--- src/runtime/virtcontainers/acrn.go | 2 +- src/runtime/virtcontainers/acrn_arch_base.go | 4 +-- .../virtcontainers/acrn_arch_base_test.go | 3 ++- src/runtime/virtcontainers/clh.go | 8 +++++- src/runtime/virtcontainers/clh_test.go | 27 +++++++++++++++++++ src/runtime/virtcontainers/qemu.go | 2 +- src/runtime/virtcontainers/qemu_amd64.go | 7 +++-- src/runtime/virtcontainers/qemu_amd64_test.go | 5 ++-- src/runtime/virtcontainers/qemu_arch_base.go | 8 +++--- .../virtcontainers/qemu_arch_base_test.go | 9 ++++++- src/runtime/virtcontainers/qemu_ppc64le.go | 7 +++-- 19 files changed, 113 insertions(+), 21 deletions(-) diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index 0ac6e14341..fd906ff0a0 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -131,6 +131,11 @@ default_maxmemory = @DEFMAXMEMSZ@ # Shared file system type: # - virtio-fs (default) # - virtio-fs-nydus +# - none +# WARNING: "none" should be carefully used, and only used in very few specific cases, as +# any update to the mount will *NOT* be reflected during the lifecycle of the pod, causing +# issues with rotation of secrets, certs, or configurations via kubernetes objects like +# configMaps or secrets, as those will be copied into the guest at *pod* *creation* *time*. shared_fs = "@DEFSHAREDFS_CLH_VIRTIOFS@" # Path to vhost-user-fs daemon. diff --git a/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in b/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in index 33574b17d5..4861cb1ed6 100644 --- a/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in +++ b/src/runtime/config/configuration-qemu-nvidia-gpu.toml.in @@ -178,6 +178,11 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # - virtio-fs (default) # - virtio-9p # - virtio-fs-nydus +# - none +# WARNING: "none" should be carefully used, and only used in very few specific cases, as +# any update to the mount will *NOT* be reflected during the lifecycle of the pod, causing +# issues with rotation of secrets, certs, or configurations via kubernetes objects like +# configMaps or secrets, as those will be copied into the guest at *pod* *creation* *time*. shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@" # Path to vhost-user-fs daemon. diff --git a/src/runtime/config/configuration-qemu-sev.toml.in b/src/runtime/config/configuration-qemu-sev.toml.in index a108b726c6..413ae1950e 100644 --- a/src/runtime/config/configuration-qemu-sev.toml.in +++ b/src/runtime/config/configuration-qemu-sev.toml.in @@ -164,6 +164,11 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # - virtio-fs (default) # - virtio-9p # - virtio-fs-nydus +# - none +# WARNING: "none" should be carefully used, and only used in very few specific cases, as +# any update to the mount will *NOT* be reflected during the lifecycle of the pod, causing +# issues with rotation of secrets, certs, or configurations via kubernetes objects like +# configMaps or secrets, as those will be copied into the guest at *pod* *creation* *time*. shared_fs = "@DEFSHAREDFS_QEMU_SEV_VIRTIOFS@" # Path to vhost-user-fs daemon. @@ -647,4 +652,4 @@ service_offload = @DEFSERVICEOFFLOAD@ # # Keys can be remotely provisioned. The Kata agent fetches them from e.g. # a HTTPS URL: -#provision=https://my-key-broker.foo/tenant/ \ No newline at end of file +#provision=https://my-key-broker.foo/tenant/ diff --git a/src/runtime/config/configuration-qemu-snp.toml.in b/src/runtime/config/configuration-qemu-snp.toml.in index 6a608a1333..54551cba52 100644 --- a/src/runtime/config/configuration-qemu-snp.toml.in +++ b/src/runtime/config/configuration-qemu-snp.toml.in @@ -176,6 +176,11 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # - virtio-fs (default) # - virtio-9p # - virtio-fs-nydus +# - none +# WARNING: "none" should be carefully used, and only used in very few specific cases, as +# any update to the mount will *NOT* be reflected during the lifecycle of the pod, causing +# issues with rotation of secrets, certs, or configurations via kubernetes objects like +# configMaps or secrets, as those will be copied into the guest at *pod* *creation* *time*. shared_fs = "@DEFSHAREDFS_QEMU_SNP_VIRTIOFS@" # Path to vhost-user-fs daemon. diff --git a/src/runtime/config/configuration-qemu-tdx.toml.in b/src/runtime/config/configuration-qemu-tdx.toml.in index 52f415f2f7..fcf1dbe3d2 100644 --- a/src/runtime/config/configuration-qemu-tdx.toml.in +++ b/src/runtime/config/configuration-qemu-tdx.toml.in @@ -172,6 +172,11 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # - virtio-fs (default) # - virtio-9p # - virtio-fs-nydus +# - none +# WARNING: "none" should be carefully used, and only used in very few specific cases, as +# any update to the mount will *NOT* be reflected during the lifecycle of the pod, causing +# issues with rotation of secrets, certs, or configurations via kubernetes objects like +# configMaps or secrets, as those will be copied into the guest at *pod* *creation* *time*. shared_fs = "@DEFSHAREDFS_QEMU_TDX_VIRTIOFS@" # Path to vhost-user-fs daemon. diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index 37ee3a7526..558d552658 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -178,6 +178,11 @@ disable_block_device_use = @DEFDISABLEBLOCK@ # - virtio-fs (default) # - virtio-9p # - virtio-fs-nydus +# - none +# WARNING: "none" should be carefully used, and only used in very few specific cases, as +# any update to the mount will *NOT* be reflected during the lifecycle of the pod, causing +# issues with rotation of secrets, certs, or configurations via kubernetes objects like +# configMaps or secrets, as those will be copied into the guest at *pod* *creation* *time*. shared_fs = "@DEFSHAREDFS_QEMU_VIRTIOFS@" # Path to vhost-user-fs daemon. diff --git a/src/runtime/pkg/device/config/config.go b/src/runtime/pkg/device/config/config.go index dee6291ed9..ed679da18a 100644 --- a/src/runtime/pkg/device/config/config.go +++ b/src/runtime/pkg/device/config/config.go @@ -81,6 +81,17 @@ const ( // VirtioFSNydus means use nydus for the shared file system VirtioFSNydus = "virtio-fs-nydus" + + // NoSharedFS means *no* shared file system solution will be used + // and files will be copied into the guest system. + // + // WARNING: This should be carefully used, and only used in very few + // specific cases, as any update to the mount will *NOT* be reflected + // during the lifecycle of the pod, causing issues with rotation of + // secrets, certs, or configurations via kubernetes objects like + // configMaps or secrets, as those will be copied into the guest at + // *pod* *creation* *time*. + NoSharedFS = "none" ) const ( diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 6b08f4afe1..a0d2322153 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -512,7 +512,7 @@ func (h hypervisor) blockDeviceAIO() (string, error) { } func (h hypervisor) sharedFS() (string, error) { - supportedSharedFS := []string{config.Virtio9P, config.VirtioFS, config.VirtioFSNydus} + supportedSharedFS := []string{config.Virtio9P, config.VirtioFS, config.VirtioFSNydus, config.NoSharedFS} if h.SharedFS == "" { return config.VirtioFS, nil @@ -1009,11 +1009,12 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { return vc.HypervisorConfig{}, err } - if sharedFS != config.VirtioFS && sharedFS != config.VirtioFSNydus { - return vc.HypervisorConfig{}, errors.New("clh only support virtio-fs or virtio-fs-nydus") + if sharedFS != config.VirtioFS && sharedFS != config.VirtioFSNydus && sharedFS != config.NoSharedFS { + return vc.HypervisorConfig{}, + fmt.Errorf("Cloud Hypervisor does not support %s shared filesystem option", sharedFS) } - if h.VirtioFSDaemon == "" { + if (sharedFS == config.VirtioFS || sharedFS == config.VirtioFSNydus) && h.VirtioFSDaemon == "" { return vc.HypervisorConfig{}, fmt.Errorf("cannot enable %s without daemon path in configuration file", sharedFS) } diff --git a/src/runtime/virtcontainers/acrn.go b/src/runtime/virtcontainers/acrn.go index 35c71a6d61..b652ff71cf 100644 --- a/src/runtime/virtcontainers/acrn.go +++ b/src/runtime/virtcontainers/acrn.go @@ -90,7 +90,7 @@ func (a *Acrn) Capabilities(ctx context.Context) types.Capabilities { span, _ := katatrace.Trace(ctx, a.Logger(), "Capabilities", acrnTracingTags, map[string]string{"sandbox_id": a.id}) defer span.End() - return a.arch.capabilities() + return a.arch.capabilities(a.config) } func (a *Acrn) HypervisorConfig() HypervisorConfig { diff --git a/src/runtime/virtcontainers/acrn_arch_base.go b/src/runtime/virtcontainers/acrn_arch_base.go index 77fb8e9e51..0b9ee53cc7 100644 --- a/src/runtime/virtcontainers/acrn_arch_base.go +++ b/src/runtime/virtcontainers/acrn_arch_base.go @@ -33,7 +33,7 @@ type acrnArch interface { kernelParameters(debug bool) []Param //capabilities returns the capabilities supported by acrn - capabilities() types.Capabilities + capabilities(config HypervisorConfig) types.Capabilities // memoryTopology returns the memory topology using the given amount of memoryMb and hostMemoryMb memoryTopology(memMb uint64) Memory @@ -361,7 +361,7 @@ func (a *acrnArchBase) memoryTopology(memoryMb uint64) Memory { return memory } -func (a *acrnArchBase) capabilities() types.Capabilities { +func (a *acrnArchBase) capabilities(config HypervisorConfig) types.Capabilities { var caps types.Capabilities caps.SetBlockDeviceSupport() diff --git a/src/runtime/virtcontainers/acrn_arch_base_test.go b/src/runtime/virtcontainers/acrn_arch_base_test.go index 61db474164..c34974e698 100644 --- a/src/runtime/virtcontainers/acrn_arch_base_test.go +++ b/src/runtime/virtcontainers/acrn_arch_base_test.go @@ -83,8 +83,9 @@ func TestAcrnArchBaseKernelParameters(t *testing.T) { func TestAcrnArchBaseCapabilities(t *testing.T) { assert := assert.New(t) acrnArchBase := newAcrnArchBase() + config := HypervisorConfig{} - c := acrnArchBase.capabilities() + c := acrnArchBase.capabilities(config) assert.True(c.IsBlockDeviceSupported()) assert.True(c.IsBlockDeviceHotplugSupported()) assert.False(c.IsFsSharingSupported()) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 6ae99d6732..9c8d1faea1 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -349,6 +349,10 @@ func (clh *cloudHypervisor) createVirtiofsDaemon(sharedPath string) (VirtiofsDae } func (clh *cloudHypervisor) setupVirtiofsDaemon(ctx context.Context) error { + if clh.config.SharedFS == config.NoSharedFS { + return nil + } + if clh.config.SharedFS == config.Virtio9P { return errors.New("cloud-hypervisor only supports virtio based file sharing") } @@ -1205,7 +1209,9 @@ func (clh *cloudHypervisor) Capabilities(ctx context.Context) types.Capabilities clh.Logger().WithField("function", "Capabilities").Info("get Capabilities") var caps types.Capabilities - caps.SetFsSharingSupport() + if clh.config.SharedFS != config.NoSharedFS { + caps.SetFsSharingSupport() + } caps.SetBlockDeviceHotplugSupport() return caps } diff --git a/src/runtime/virtcontainers/clh_test.go b/src/runtime/virtcontainers/clh_test.go index b5c800e956..a5eacabcab 100644 --- a/src/runtime/virtcontainers/clh_test.go +++ b/src/runtime/virtcontainers/clh_test.go @@ -726,3 +726,30 @@ func TestClhSetConfig(t *testing.T) { assert.Equal(clh.config, config) } + +func TestClhCapabilities(t *testing.T) { + assert := assert.New(t) + + hConfig, err := newClhConfig() + assert.NoError(err) + + clh := &cloudHypervisor{} + assert.Equal(clh.config, HypervisorConfig{}) + + hConfig.SharedFS = config.VirtioFS + + err = clh.setConfig(&hConfig) + assert.NoError(err) + + var ctx context.Context + c := clh.Capabilities(ctx) + assert.True(c.IsFsSharingSupported()) + + hConfig.SharedFS = config.NoSharedFS + + err = clh.setConfig(&hConfig) + assert.NoError(err) + + c = clh.Capabilities(ctx) + assert.False(c.IsFsSharingSupported()) +} diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index d856490e5b..5ad55121cb 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -211,7 +211,7 @@ func (q *qemu) Capabilities(ctx context.Context) types.Capabilities { span, _ := katatrace.Trace(ctx, q.Logger(), "Capabilities", qemuTracingTags, map[string]string{"sandbox_id": q.id}) defer span.End() - return q.arch.capabilities() + return q.arch.capabilities(q.config) } func (q *qemu) HypervisorConfig() HypervisorConfig { diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go index 4ecf0804a4..dc47f17cc8 100644 --- a/src/runtime/virtcontainers/qemu_amd64.go +++ b/src/runtime/virtcontainers/qemu_amd64.go @@ -16,6 +16,7 @@ import ( "github.com/sirupsen/logrus" "github.com/intel-go/cpuid" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" ) @@ -155,7 +156,7 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) { return q, nil } -func (q *qemuAmd64) capabilities() types.Capabilities { +func (q *qemuAmd64) capabilities(hConfig HypervisorConfig) types.Capabilities { var caps types.Capabilities if q.qemuMachine.Type == QemuQ35 || @@ -164,7 +165,9 @@ func (q *qemuAmd64) capabilities() types.Capabilities { } caps.SetMultiQueueSupport() - caps.SetFsSharingSupport() + if hConfig.SharedFS != config.NoSharedFS { + caps.SetFsSharingSupport() + } return caps } diff --git a/src/runtime/virtcontainers/qemu_amd64_test.go b/src/runtime/virtcontainers/qemu_amd64_test.go index cb9a7bb388..82005b7459 100644 --- a/src/runtime/virtcontainers/qemu_amd64_test.go +++ b/src/runtime/virtcontainers/qemu_amd64_test.go @@ -42,13 +42,14 @@ func TestQemuAmd64BadMachineType(t *testing.T) { func TestQemuAmd64Capabilities(t *testing.T) { assert := assert.New(t) + config := HypervisorConfig{} amd64 := newTestQemu(assert, QemuQ35) - caps := amd64.capabilities() + caps := amd64.capabilities(config) assert.True(caps.IsBlockDeviceHotplugSupported()) amd64 = newTestQemu(assert, QemuMicrovm) - caps = amd64.capabilities() + caps = amd64.capabilities(config) assert.False(caps.IsBlockDeviceHotplugSupported()) } diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index 9aff1e76c2..72c9853ceb 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -51,7 +51,7 @@ type qemuArch interface { kernelParameters(debug bool) []Param //capabilities returns the capabilities supported by QEMU - capabilities() types.Capabilities + capabilities(config HypervisorConfig) types.Capabilities // bridges sets the number bridges for the machine type bridges(number uint32) @@ -280,11 +280,13 @@ func (q *qemuArchBase) kernelParameters(debug bool) []Param { return params } -func (q *qemuArchBase) capabilities() types.Capabilities { +func (q *qemuArchBase) capabilities(hConfig HypervisorConfig) types.Capabilities { var caps types.Capabilities caps.SetBlockDeviceHotplugSupport() caps.SetMultiQueueSupport() - caps.SetFsSharingSupport() + if hConfig.SharedFS != config.NoSharedFS { + caps.SetFsSharingSupport() + } return caps } diff --git a/src/runtime/virtcontainers/qemu_arch_base_test.go b/src/runtime/virtcontainers/qemu_arch_base_test.go index 51c11bd91d..b0d08ddc92 100644 --- a/src/runtime/virtcontainers/qemu_arch_base_test.go +++ b/src/runtime/virtcontainers/qemu_arch_base_test.go @@ -117,9 +117,16 @@ func TestQemuArchBaseKernelParameters(t *testing.T) { func TestQemuArchBaseCapabilities(t *testing.T) { assert := assert.New(t) qemuArchBase := newQemuArchBase() + hConfig := HypervisorConfig{} + hConfig.SharedFS = config.VirtioFS - c := qemuArchBase.capabilities() + c := qemuArchBase.capabilities(hConfig) assert.True(c.IsBlockDeviceHotplugSupported()) + assert.True(c.IsFsSharingSupported()) + + hConfig.SharedFS = config.NoSharedFS + c = qemuArchBase.capabilities(hConfig) + assert.False(c.IsFsSharingSupported()) } func TestQemuArchBaseBridges(t *testing.T) { diff --git a/src/runtime/virtcontainers/qemu_ppc64le.go b/src/runtime/virtcontainers/qemu_ppc64le.go index 2d4010fbc5..7d71d72ba5 100644 --- a/src/runtime/virtcontainers/qemu_ppc64le.go +++ b/src/runtime/virtcontainers/qemu_ppc64le.go @@ -11,6 +11,7 @@ import ( "fmt" "time" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/sirupsen/logrus" @@ -97,7 +98,7 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) { return q, nil } -func (q *qemuPPC64le) capabilities() types.Capabilities { +func (q *qemuPPC64le) capabilities(hConfig HypervisorConfig) types.Capabilities { var caps types.Capabilities // pseries machine type supports hotplugging drives @@ -106,7 +107,9 @@ func (q *qemuPPC64le) capabilities() types.Capabilities { } caps.SetMultiQueueSupport() - caps.SetFsSharingSupport() + if hConfig.SharedFS != config.NoSharedFS { + caps.SetFsSharingSupport() + } return caps }