Subject: [PATCH] qemu: add annotations for iommu_platform

for s390x virtio devices

Add iommu_platform annotations for qemu for ccw,
other supported devices can also make use of that.

  Fixes #603

Signed-off-by: Qi Feng Huo <huoqif@cn.ibm.com>

src/runtime/cli/config/configuration-qemu-virtiofs.toml.in

@@ -195,6 +195,10 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@"
 # command line: intel_iommu=on,iommu=pt
 #enable_iommu = true
 
+# Enable IOMMU_PLATFORM, default false
+# Enabling this will result in the VM device having iommu_platform=on set
+#enable_iommu_platform = true
+
 # Enable file based guest memory support. The default is an empty string which
 # will disable this feature. In the case of virtio-fs, this is enabled
 # automatically and '/dev/shm' is used as the backing folder.

src/runtime/cli/config/configuration-qemu.toml.in

@@ -201,6 +201,10 @@ vhost_user_store_path = "@DEFVHOSTUSERSTOREPATH@"
 # command line: intel_iommu=on,iommu=pt
 #enable_iommu = true
 
+# Enable IOMMU_PLATFORM, default false
+# Enabling this will result in the VM device having iommu_platform=on set
+#enable_iommu_platform = true
+
 # Enable file based guest memory support. The default is an empty string which
 # will disable this feature. In the case of virtio-fs, this is enabled
 # automatically and '/dev/shm' is used as the backing folder.

src/runtime/pkg/katautils/config-settings.go.in

@@ -40,6 +40,7 @@ const defaultEnableIOThreads bool = false
 const defaultEnableMemPrealloc bool = false
 const defaultEnableHugePages bool = false
 const defaultEnableIOMMU bool = false
+const defaultEnableIOMMUPlatform bool = false
 const defaultFileBackedMemRootDir string = ""
 const defaultEnableSwap bool = false
 const defaultEnableDebug bool = false

src/runtime/pkg/katautils/config.go

@@ -106,6 +106,7 @@ type hypervisor struct {
 	HugePages               bool     `toml:"enable_hugepages"`
 	VirtioMem               bool     `toml:"enable_virtio_mem"`
 	IOMMU                   bool     `toml:"enable_iommu"`
+	IOMMUPlatform           bool     `toml:"enable_iommu_platform"`
 	FileBackedMemRootDir    string   `toml:"file_mem_backend"`
 	Swap                    bool     `toml:"enable_swap"`
 	Debug                   bool     `toml:"enable_debug"`
@@ -431,6 +432,15 @@ func (h hypervisor) getTxRateLimiterCfg() (uint64, error) {
 	return h.TxRateLimiterMaxRate, nil
 }
 
+func (h hypervisor) getIOMMUPlatform() bool {
+	if h.IOMMUPlatform {
+		kataUtilsLogger.Info("IOMMUPlatform is enabled by default.")
+	} else {
+		kataUtilsLogger.Info("IOMMUPlatform is disabled by default.")
+	}
+	return h.IOMMUPlatform
+}
+
 func (a agent) debug() bool {
 	return a.Debug
 }
@@ -638,6 +648,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		MemPrealloc:             h.MemPrealloc,
 		HugePages:               h.HugePages,
 		IOMMU:                   h.IOMMU,
+		IOMMUPlatform:           h.getIOMMUPlatform(),
 		FileBackedMemRootDir:    h.FileBackedMemRootDir,
 		Mlock:                   !h.Swap,
 		Debug:                   h.Debug,
@@ -987,6 +998,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		MemPrealloc:             defaultEnableMemPrealloc,
 		HugePages:               defaultEnableHugePages,
 		IOMMU:                   defaultEnableIOMMU,
+		IOMMUPlatform:           defaultEnableIOMMUPlatform,
 		FileBackedMemRootDir:    defaultFileBackedMemRootDir,
 		Mlock:                   !defaultEnableSwap,
 		Debug:                   defaultEnableDebug,

src/runtime/virtcontainers/hypervisor.go

@@ -358,6 +358,9 @@ type HypervisorConfig struct {
 	// IOMMU specifies if the VM should have a vIOMMU
 	IOMMU bool
 
+	// IOMMUPlatform is used to indicate if IOMMU_PLATFORM is enabled for supported devices
+	IOMMUPlatform bool
+
 	// Realtime Used to enable/disable realtime
 	Realtime bool
 

src/runtime/virtcontainers/pkg/annotations/annotations.go

@@ -151,6 +151,9 @@ const (
 	// Iommu is a sandbox annotation to specify if the VM should have a vIOMMU device
 	IOMMU = kataAnnotHypervisorPrefix + "enable_iommu"
 
+	// Enable Hypervisor Devices IOMMU_PLATFORM
+	IOMMUPlatform = kataAnnotHypervisorPrefix + "enable_iommu_platform"
+
 	// FileBackedMemRootDir is a sandbox annotation to soecify file based memory backend root directory
 	FileBackedMemRootDir = kataAnnotHypervisorPrefix + "file_mem_backend"
 

src/runtime/virtcontainers/pkg/oci/utils.go

@@ -530,6 +530,15 @@ func addHypervisorMemoryOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig
 
 		sbConfig.HypervisorConfig.IOMMU = iommu
 	}
+
+	if value, ok := ocispec.Annotations[vcAnnotations.IOMMUPlatform]; ok {
+		deviceIOMMU, err := strconv.ParseBool(value)
+		if err != nil {
+			return fmt.Errorf("Error parsing annotation for enable_iommu_platform: Please specify boolean value 'true|false'")
+		}
+
+		sbConfig.HypervisorConfig.IOMMUPlatform = deviceIOMMU
+	}
 	return nil
 }
 

src/runtime/virtcontainers/pkg/oci/utils_test.go

@@ -785,6 +785,7 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	ocispec.Annotations[vcAnnotations.HotplugVFIOOnRootBus] = "true"
 	ocispec.Annotations[vcAnnotations.PCIeRootPort] = "2"
 	ocispec.Annotations[vcAnnotations.EntropySource] = "/dev/urandom"
+	ocispec.Annotations[vcAnnotations.IOMMUPlatform] = "true"
 	// 10Mbit
 	ocispec.Annotations[vcAnnotations.RxRateLimiterMaxRate] = "10000000"
 	ocispec.Annotations[vcAnnotations.TxRateLimiterMaxRate] = "10000000"
@@ -820,6 +821,7 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	assert.Equal(config.HypervisorConfig.HotplugVFIOOnRootBus, true)
 	assert.Equal(config.HypervisorConfig.PCIeRootPort, uint32(2))
 	assert.Equal(config.HypervisorConfig.EntropySource, "/dev/urandom")
+	assert.Equal(config.HypervisorConfig.IOMMUPlatform, true)
 	assert.Equal(config.HypervisorConfig.RxRateLimiterMaxRate, uint64(10000000))
 	assert.Equal(config.HypervisorConfig.TxRateLimiterMaxRate, uint64(10000000))
 

src/runtime/virtcontainers/qemu.go

@@ -478,15 +478,16 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa
 	}
 
 	knobs := govmmQemu.Knobs{
-		NoUserConfig: true,
-		NoDefaults:   true,
-		NoGraphic:    true,
-		NoReboot:     true,
-		Daemonize:    true,
-		MemPrealloc:  q.config.MemPrealloc,
-		HugePages:    q.config.HugePages,
-		Realtime:     q.config.Realtime,
-		Mlock:        q.config.Mlock,
+		NoUserConfig:  true,
+		NoDefaults:    true,
+		NoGraphic:     true,
+		NoReboot:      true,
+		Daemonize:     true,
+		MemPrealloc:   q.config.MemPrealloc,
+		HugePages:     q.config.HugePages,
+		Realtime:      q.config.Realtime,
+		Mlock:         q.config.Mlock,
+		IOMMUPlatform: q.config.IOMMUPlatform,
 	}
 
 	kernelPath, err := q.config.KernelAssetPath()