diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in index 502dfb43ad..e98112ea8c 100644 --- a/cli/config/configuration-qemu.toml.in +++ b/cli/config/configuration-qemu.toml.in @@ -165,6 +165,12 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # result in memory pre allocation #enable_hugepages = true +# Enable file based guest memory support. The default is an empty string which +# will disable this feature. In the case of virtio-fs, this is enabled +# automatically and '/dev/shm' is used as the backing folder. +# This option will be ignored if VM templating is enabled. +#file_mem_backend = "" + # Enable swap of vm memory. Default false. # The behaviour is undefined if mem_prealloc is also set to true #enable_swap = true diff --git a/pkg/katautils/config-settings.go b/pkg/katautils/config-settings.go index d1fab1b4db..7c13339552 100644 --- a/pkg/katautils/config-settings.go +++ b/pkg/katautils/config-settings.go @@ -35,6 +35,7 @@ const defaultBlockDeviceCacheNoflush bool = false const defaultEnableIOThreads bool = false const defaultEnableMemPrealloc bool = false const defaultEnableHugePages bool = false +const defaultFileBackedMemRootDir string = "" const defaultEnableSwap bool = false const defaultEnableDebug bool = false const defaultDisableNestingChecks bool = false diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index 8567f78bf3..5b6f7ea181 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -109,6 +109,7 @@ type hypervisor struct { DisableBlockDeviceUse bool `toml:"disable_block_device_use"` MemPrealloc bool `toml:"enable_mem_prealloc"` HugePages bool `toml:"enable_hugepages"` + FileBackedMemRootDir string `toml:"file_mem_backend"` Swap bool `toml:"enable_swap"` Debug bool `toml:"enable_debug"` DisableNestingChecks bool `toml:"disable_nesting_checks"` @@ -584,6 +585,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { VirtioFSCache: h.VirtioFSCache, MemPrealloc: h.MemPrealloc, HugePages: h.HugePages, + FileBackedMemRootDir: h.FileBackedMemRootDir, Mlock: !h.Swap, Debug: h.Debug, DisableNestingChecks: h.DisableNestingChecks, @@ -850,6 +852,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig { DefaultBridges: defaultBridgesCount, MemPrealloc: defaultEnableMemPrealloc, HugePages: defaultEnableHugePages, + FileBackedMemRootDir: defaultFileBackedMemRootDir, Mlock: !defaultEnableSwap, Debug: defaultEnableDebug, DisableNestingChecks: defaultDisableNestingChecks, diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index d94215de7a..266c409838 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -263,6 +263,9 @@ type HypervisorConfig struct { // HugePages specifies if the memory should be pre-allocated from huge pages HugePages bool + // File based memory backend root directory + FileBackedMemRootDir string + // Realtime Used to enable/disable realtime Realtime bool diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go index 3da8c5e731..1d10ed483c 100644 --- a/virtcontainers/persist/api/config.go +++ b/virtcontainers/persist/api/config.go @@ -121,6 +121,9 @@ type HypervisorConfig struct { // HugePages specifies if the memory should be pre-allocated from huge pages HugePages bool + // File based memory backend root directory + FileBackedMemRootDir string + // Realtime Used to enable/disable realtime Realtime bool diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 1217dfc32b..0530613e59 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -96,9 +96,10 @@ const ( qmpCapErrMsg = "Failed to negoatiate QMP capabilities" qmpExecCatCmd = "exec:cat" - scsiControllerID = "scsi0" - rngID = "rng0" - vsockKernelOption = "agent.use_vsock" + scsiControllerID = "scsi0" + rngID = "rng0" + vsockKernelOption = "agent.use_vsock" + fallbackFileBackedMemDir = "/dev/shm" ) var qemuMajorVersion int @@ -423,6 +424,23 @@ func (q *qemu) setupTemplate(knobs *govmmQemu.Knobs, memory *govmmQemu.Memory) g return incoming } +func (q *qemu) setupFileBackedMem(knobs *govmmQemu.Knobs, memory *govmmQemu.Memory) { + var target string + if q.config.FileBackedMemRootDir != "" { + target = q.config.FileBackedMemRootDir + } else { + target = fallbackFileBackedMemDir + } + if _, err := os.Stat(target); err != nil { + q.Logger().WithError(err).Error("File backed memory location does not exist") + return + } + + knobs.FileBackedMem = true + knobs.FileBackedMemShared = true + memory.Path = target +} + // createSandbox is the Hypervisor sandbox creation implementation for govmmQemu. func (q *qemu) createSandbox(ctx context.Context, id string, hypervisorConfig *HypervisorConfig, store *store.VCStore) error { // Save the tracing context @@ -476,6 +494,19 @@ func (q *qemu) createSandbox(ctx context.Context, id string, hypervisorConfig *H incoming := q.setupTemplate(&knobs, &memory) + // With the current implementations, VM templating will not work with file + // based memory (stand-alone) or virtiofs. This is because VM templating + // builds the first VM with file-backed memory and shared=on and the + // subsequent ones with shared=off. virtio-fs always requires shared=on for + // memory. + if q.config.SharedFS == config.VirtioFS || q.config.FileBackedMemRootDir != "" { + if !(q.config.BootToBeTemplate || q.config.BootFromTemplate) { + q.setupFileBackedMem(&knobs, &memory) + } else { + return errors.New("VM templating has been enabled with either virtio-fs or file backed memory and this configuration will not work") + } + } + rtc := govmmQemu.RTC{ Base: "utc", DriftFix: "slew", diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index cf073134fc..d1e95f4ea9 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -16,6 +16,7 @@ import ( "testing" govmmQemu "github.com/intel/govmm/qemu" + "github.com/kata-containers/runtime/virtcontainers/device/config" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/kata-containers/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" @@ -482,3 +483,70 @@ func TestQemuAddDeviceToBridge(t *testing.T) { exceptErr = errors.New("failed to get available address from bridges") assert.Equal(exceptErr, err) } + +func TestQemuFileBackedMem(t *testing.T) { + assert := assert.New(t) + + // Check default Filebackedmem location for virtio-fs + sandbox, err := createQemuSandboxConfig() + if err != nil { + t.Fatal(err) + } + q := &qemu{} + sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS + if err = q.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil { + t.Fatal(err) + } + assert.Equal(q.qemuConfig.Knobs.FileBackedMem, true) + assert.Equal(q.qemuConfig.Knobs.FileBackedMemShared, true) + assert.Equal(q.qemuConfig.Memory.Path, fallbackFileBackedMemDir) + + // Check failure for VM templating + sandbox, err = createQemuSandboxConfig() + if err != nil { + t.Fatal(err) + } + q = &qemu{} + sandbox.config.HypervisorConfig.BootToBeTemplate = true + sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS + sandbox.config.HypervisorConfig.MemoryPath = fallbackFileBackedMemDir + + err = q.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store) + + expectErr := errors.New("VM templating has been enabled with either virtio-fs or file backed memory and this configuration will not work") + assert.Equal(expectErr, err) + + // Check Setting of non-existent shared-mem path + sandbox, err = createQemuSandboxConfig() + if err != nil { + t.Fatal(err) + } + q = &qemu{} + sandbox.config.HypervisorConfig.FileBackedMemRootDir = "/tmp/xyzabc" + if err = q.createSandbox(context.Background(), sandbox.id, &sandbox.config.HypervisorConfig, sandbox.store); err != nil { + t.Fatal(err) + } + assert.Equal(q.qemuConfig.Knobs.FileBackedMem, false) + assert.Equal(q.qemuConfig.Knobs.FileBackedMemShared, false) + assert.Equal(q.qemuConfig.Memory.Path, "") +} + +func createQemuSandboxConfig() (*Sandbox, error) { + + qemuConfig := newQemuConfig() + sandbox := Sandbox{ + ctx: context.Background(), + id: "testSandbox", + config: &SandboxConfig{ + HypervisorConfig: qemuConfig, + }, + } + + vcStore, err := store.NewVCSandboxStore(sandbox.ctx, sandbox.id) + if err != nil { + return &Sandbox{}, err + } + sandbox.store = vcStore + + return &sandbox, nil +}