qemu: add disalbe_image_nvdimm option

To control whether an image rootfs is used as nvdimm device or just
plain virtio-block device.

Fixes: #2372
Signed-off-by: Peng Tao <bergwolf@hyper.sh>

cli/config/configuration-qemu-virtiofs.toml.in

@@ -205,6 +205,11 @@ enable_iothreads = @DEFENABLEIOTHREADS@
 # Default false
 #use_vsock = true
 
+# If false and nvdimm is supported, use nvdimm device to plug guest image.
+# Otherwise virtio-block device is used.
+# Default false
+#disable_image_nvdimm = true
+
 # VFIO devices are hotplugged on a bridge by default.
 # Enable hotplugging on root bus. This may be required for devices with
 # a large PCI bar, as this is a current limitation with hotplugging on

cli/config/configuration-qemu.toml.in

@@ -206,6 +206,11 @@ enable_iothreads = @DEFENABLEIOTHREADS@
 # Default false
 #use_vsock = true
 
+# If false and nvdimm is supported, use nvdimm device to plug guest image.
+# Otherwise virtio-block device is used.
+# Default is false
+#disable_image_nvdimm = true
+
 # VFIO devices are hotplugged on a bridge by default. 
 # Enable hotplugging on root bus. This may be required for devices with
 # a large PCI bar, as this is a current limitation with hotplugging on 

pkg/katautils/config-settings.go

@@ -46,6 +46,7 @@ const defaultHotplugVFIOOnRootBus bool = false
 const defaultEntropySource = "/dev/urandom"
 const defaultGuestHookPath string = ""
 const defaultVirtioFSCacheMode = "none"
+const defaultDisableImageNvdimm = false
 
 const defaultTemplatePath string = "/run/vc/vm/template"
 const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock"

pkg/katautils/config.go

@@ -120,6 +120,7 @@ type hypervisor struct {
 	DisableNestingChecks    bool     `toml:"disable_nesting_checks"`
 	EnableIOThreads         bool     `toml:"enable_iothreads"`
 	UseVSock                bool     `toml:"use_vsock"`
+	DisableImageNvdimm      bool     `toml:"disable_image_nvdimm"`
 	HotplugVFIOOnRootBus    bool     `toml:"hotplug_vfio_on_root_bus"`
 	DisableVhostNet         bool     `toml:"disable_vhost_net"`
 	GuestHookPath           string   `toml:"guest_hook_path"`
@@ -643,6 +644,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		EnableIOThreads:         h.EnableIOThreads,
 		Msize9p:                 h.msize9p(),
 		UseVSock:                useVSock,
+		DisableImageNvdimm:      h.DisableImageNvdimm,
 		HotplugVFIOOnRootBus:    h.HotplugVFIOOnRootBus,
 		DisableVhostNet:         h.DisableVhostNet,
 		GuestHookPath:           h.guestHookPath(),
@@ -1069,6 +1071,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		HotplugVFIOOnRootBus:    defaultHotplugVFIOOnRootBus,
 		GuestHookPath:           defaultGuestHookPath,
 		VirtioFSCache:           defaultVirtioFSCacheMode,
+		DisableImageNvdimm:      defaultDisableImageNvdimm,
 	}
 }
 

virtcontainers/hypervisor.go

@@ -90,6 +90,13 @@ var commonNvdimmKernelRootParams = []Param{ //nolint: unused, deadcode, varcheck
 	{"rootfstype", "ext4"},
 }
 
+// agnostic list of kernel root parameters for NVDIMM
+var commonNvdimmNoDAXKernelRootParams = []Param{ //nolint: unused, deadcode, varcheck
+	{"root", "/dev/pmem0p1"},
+	{"rootflags", "data=ordered,errors=remount-ro ro"},
+	{"rootfstype", "ext4"},
+}
+
 // agnostic list of kernel root parameters for virtio-blk
 var commonVirtioblkKernelRootParams = []Param{ //nolint: unused, deadcode, varcheck
 	{"root", "/dev/vda1"},
@@ -352,6 +359,9 @@ type HypervisorConfig struct {
 	// UseVSock use a vsock for agent communication
 	UseVSock bool
 
+	// DisableImageNvdimm is used to disable guest rootfs image nvdimm devices
+	DisableImageNvdimm bool
+
 	// HotplugVFIOOnRootBus is used to indicate if devices need to be hotplugged on the
 	// root bus instead of a bridge.
 	HotplugVFIOOnRootBus bool

virtcontainers/persist.go

@@ -241,6 +241,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
 		Mlock:                   sconfig.HypervisorConfig.Mlock,
 		DisableNestingChecks:    sconfig.HypervisorConfig.DisableNestingChecks,
 		UseVSock:                sconfig.HypervisorConfig.UseVSock,
+		DisableImageNvdimm:      sconfig.HypervisorConfig.DisableImageNvdimm,
 		HotplugVFIOOnRootBus:    sconfig.HypervisorConfig.HotplugVFIOOnRootBus,
 		BootToBeTemplate:        sconfig.HypervisorConfig.BootToBeTemplate,
 		BootFromTemplate:        sconfig.HypervisorConfig.BootFromTemplate,
@@ -532,6 +533,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
 		Mlock:                   hconf.Mlock,
 		DisableNestingChecks:    hconf.DisableNestingChecks,
 		UseVSock:                hconf.UseVSock,
+		DisableImageNvdimm:      hconf.DisableImageNvdimm,
 		HotplugVFIOOnRootBus:    hconf.HotplugVFIOOnRootBus,
 		BootToBeTemplate:        hconf.BootToBeTemplate,
 		BootFromTemplate:        hconf.BootFromTemplate,

virtcontainers/persist/api/config.go

@@ -142,6 +142,9 @@ type HypervisorConfig struct {
 	// UseVSock use a vsock for agent communication
 	UseVSock bool
 
+	// DisableImageNvdimm disables nvdimm for guest rootfs image
+	DisableImageNvdimm bool
+
 	// HotplugVFIOOnRootBus is used to indicate if devices need to be hotplugged on the
 	// root bus instead of a bridge.
 	HotplugVFIOOnRootBus bool

virtcontainers/qemu.go

@@ -248,7 +248,7 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig, vcStore *sto
 	if err != nil {
 		return err
 	}
-	if initrdPath == "" && imagePath != "" {
+	if initrdPath == "" && imagePath != "" && !q.config.DisableImageNvdimm {
 		q.nvdimmCount = 1
 	} else {
 		q.nvdimmCount = 0

virtcontainers/qemu_amd64.go

@@ -6,7 +6,6 @@
 package virtcontainers
 
 import (
-	"strings"
 	"time"
 
 	"github.com/kata-containers/runtime/virtcontainers/types"
@@ -25,8 +24,6 @@ const defaultQemuPath = "/usr/bin/qemu-system-x86_64"
 
 const defaultQemuMachineType = QemuPC
 
-const qemuNvdimmOption = "nvdimm"
-
 const defaultQemuMachineOptions = "accel=kvm,kernel_irqchip"
 
 const qmpMigrationWaitTimeout = 5 * time.Second
@@ -37,8 +34,6 @@ var qemuPaths = map[string]string{
 	QemuQ35:    defaultQemuPath,
 }
 
-var kernelRootParams = commonNvdimmKernelRootParams
-
 var kernelParams = []Param{
 	{"tsc", "reliable"},
 	{"no_timer_check", ""},
@@ -101,19 +96,12 @@ func newQemuArch(config HypervisorConfig) qemuArch {
 			kernelParamsNonDebug:  kernelParamsNonDebug,
 			kernelParamsDebug:     kernelParamsDebug,
 			kernelParams:          kernelParams,
+			disableNvdimm:         config.DisableImageNvdimm,
+			dax:                   true,
 		},
 		vmFactory: factory,
 	}
 
-	if config.ImagePath != "" {
-		for i := range q.supportedQemuMachines {
-			q.supportedQemuMachines[i].Options = strings.Join([]string{
-				q.supportedQemuMachines[i].Options,
-				qemuNvdimmOption,
-			}, ",")
-		}
-	}
-
 	q.handleImagePath(config)
 
 	return q
@@ -158,7 +146,10 @@ func (q *qemuAmd64) memoryTopology(memoryMb, hostMemoryMb uint64, slots uint8) g
 }
 
 func (q *qemuAmd64) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) {
-	return q.appendNvdimmImage(devices, path)
+	if !q.disableNvdimm {
+		return q.appendNvdimmImage(devices, path)
+	}
+	return q.appendBlockImage(devices, path)
 }
 
 // appendBridges appends to devices the given bridges

virtcontainers/qemu_amd64_test.go

@@ -114,7 +114,6 @@ func TestQemuAmd64MemoryTopology(t *testing.T) {
 }
 
 func TestQemuAmd64AppendImage(t *testing.T) {
-	var devices []govmmQemu.Device
 	assert := assert.New(t)
 
 	f, err := ioutil.TempFile("", "img")
@@ -131,6 +130,7 @@ func TestQemuAmd64AppendImage(t *testing.T) {
 
 	cfg := qemuConfig(QemuPC)
 	cfg.ImagePath = f.Name()
+	cfg.DisableImageNvdimm = false
 	amd64 := newQemuArch(cfg)
 	for _, m := range amd64.(*qemuAmd64).supportedQemuMachines {
 		assert.Contains(m.Options, qemuNvdimmOption)
@@ -147,12 +147,33 @@ func TestQemuAmd64AppendImage(t *testing.T) {
 		},
 	}
 
-	devices, err = amd64.appendImage(devices, f.Name())
+	devices, err := amd64.appendImage(nil, f.Name())
 	assert.NoError(err)
 	assert.Equal(expectedOut, devices)
 
 	// restore default supportedQemuMachines options
 	assert.Equal(len(supportedQemuMachines), copy(supportedQemuMachines, machinesCopy))
+
+	cfg.DisableImageNvdimm = true
+	amd64 = newQemuArch(cfg)
+	for _, m := range amd64.(*qemuAmd64).supportedQemuMachines {
+		assert.NotContains(m.Options, qemuNvdimmOption)
+	}
+
+	found := false
+	devices, err = amd64.appendImage(nil, f.Name())
+	assert.NoError(err)
+	for _, d := range devices {
+		if b, ok := d.(govmmQemu.BlockDevice); ok {
+			assert.Equal(b.Driver, govmmQemu.VirtioBlock)
+			assert.True(b.ShareRW)
+			found = true
+		}
+	}
+	assert.True(found)
+
+	// restore default supportedQemuMachines options
+	assert.Equal(len(supportedQemuMachines), copy(supportedQemuMachines, machinesCopy))
 }
 
 func TestQemuAmd64AppendBridges(t *testing.T) {

virtcontainers/qemu_arch_base.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"os"
 	"strconv"
+	"strings"
 
 	govmmQemu "github.com/intel/govmm/qemu"
 
@@ -133,6 +134,8 @@ type qemuArchBase struct {
 	memoryOffset          uint32
 	nestedRun             bool
 	vhost                 bool
+	disableNvdimm         bool
+	dax                   bool
 	networkIndex          int
 	qemuPaths             map[string]string
 	supportedQemuMachines []govmmQemu.Machine
@@ -178,6 +181,8 @@ const (
 	QemuCCWVirtio = "s390-ccw-virtio"
 
 	qmpCapMigrationIgnoreShared = "x-ignore-shared"
+
+	qemuNvdimmOption = "nvdimm"
 )
 
 // kernelParamsNonDebug is a list of the default kernel
@@ -660,6 +665,20 @@ func (q *qemuArchBase) appendRNGDevice(devices []govmmQemu.Device, rngDev config
 
 func (q *qemuArchBase) handleImagePath(config HypervisorConfig) {
 	if config.ImagePath != "" {
+		kernelRootParams := commonVirtioblkKernelRootParams
+		if !q.disableNvdimm {
+			for i := range q.supportedQemuMachines {
+				q.supportedQemuMachines[i].Options = strings.Join([]string{
+					q.supportedQemuMachines[i].Options,
+					qemuNvdimmOption,
+				}, ",")
+			}
+			if q.dax {
+				kernelRootParams = commonNvdimmKernelRootParams
+			} else {
+				kernelRootParams = commonNvdimmNoDAXKernelRootParams
+			}
+		}
 		q.kernelParams = append(q.kernelParams, kernelRootParams...)
 		q.kernelParamsNonDebug = append(q.kernelParamsNonDebug, kernelParamsSystemdNonDebug...)
 		q.kernelParamsDebug = append(q.kernelParamsDebug, kernelParamsSystemdDebug...)

virtcontainers/qemu_arm64.go

@@ -25,8 +25,6 @@ const defaultQemuPath = "/usr/bin/qemu-system-aarch64"
 
 const defaultQemuMachineType = QemuVirt
 
-const qemuNvdimmOption = "nvdimm"
-
 const qmpMigrationWaitTimeout = 10 * time.Second
 
 var defaultQemuMachineOptions = "usb=off,accel=kvm,gic-version=" + getGuestGICVersion()
@@ -41,15 +39,6 @@ var kernelParams = []Param{
 	{"iommu.passthrough", "0"},
 }
 
-// For now, AArch64 doesn't support DAX, so we couldn't use
-// commonNvdimmKernelRootParams, the agnostic list of kernel
-// root parameters for NVDIMM
-var kernelRootParams = []Param{
-	{"root", "/dev/pmem0p1"},
-	{"rootflags", "data=ordered,errors=remount-ro ro"},
-	{"rootfstype", "ext4"},
-}
-
 var supportedQemuMachines = []govmmQemu.Machine{
 	{
 		Type:    QemuVirt,
@@ -150,20 +139,11 @@ func newQemuArch(config HypervisorConfig) qemuArch {
 			kernelParamsNonDebug:  kernelParamsNonDebug,
 			kernelParamsDebug:     kernelParamsDebug,
 			kernelParams:          kernelParams,
+			disableNvdimm:         config.DisableImageNvdimm,
 		},
 	}
 
-	if config.ImagePath != "" {
+	q.handleImagePath(config)
-		for i := range q.supportedQemuMachines {
-			q.supportedQemuMachines[i].Options = strings.Join([]string{
-				q.supportedQemuMachines[i].Options,
-				qemuNvdimmOption,
-			}, ",")
-		}
-		q.kernelParams = append(q.kernelParams, kernelRootParams...)
-		q.kernelParamsNonDebug = append(q.kernelParamsNonDebug, kernelParamsSystemdNonDebug...)
-		q.kernelParamsDebug = append(q.kernelParamsDebug, kernelParamsSystemdDebug...)
-	}
 
 	return q
 }
@@ -178,7 +158,10 @@ func (q *qemuArm64) appendBridges(devices []govmmQemu.Device) []govmmQemu.Device
 }
 
 func (q *qemuArm64) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) {
-	return q.appendNvdimmImage(devices, path)
+	if !q.disableNvdimm {
+		return q.appendNvdimmImage(devices, path)
+	}
+	return q.appendBlockImage(devices, path)
 }
 
 func (q *qemuArm64) setIgnoreSharedMemoryMigrationCaps(_ context.Context, _ *govmmQemu.QMP) error {

virtcontainers/qemu_ppc64le.go

@@ -32,8 +32,6 @@ var qemuPaths = map[string]string{
 	QemuPseries: defaultQemuPath,
 }
 
-var kernelRootParams = []Param{}
-
 var kernelParams = []Param{
 	{"tsc", "reliable"},
 	{"no_timer_check", ""},

virtcontainers/qemu_s390x.go

@@ -38,8 +38,6 @@ var kernelParams = []Param{
 	{"console", "ttysclp0"},
 }
 
-var kernelRootParams = commonVirtioblkKernelRootParams
-
 var ccwbridge = types.NewBridge(types.CCW, "", make(map[uint32]string, types.CCWBridgeMaxCapacity), 0)
 
 var supportedQemuMachines = []govmmQemu.Machine{
@@ -78,7 +76,7 @@ func newQemuArch(config HypervisorConfig) qemuArch {
 	q.Bridges = append(q.Bridges, ccwbridge)
 
 	if config.ImagePath != "" {
-		q.kernelParams = append(q.kernelParams, kernelRootParams...)
+		q.kernelParams = append(q.kernelParams, commonVirtioblkKernelRootParams...)
 		q.kernelParamsNonDebug = append(q.kernelParamsNonDebug, kernelParamsSystemdNonDebug...)
 		q.kernelParamsDebug = append(q.kernelParamsDebug, kernelParamsSystemdDebug...)
 	}