qemu: add cpu_features option

[ port from runtime commit 0100af18a2afdd6dfcc95129ec6237ba4915b3e5 ]

To control whether guest can enable/disable some CPU features. E.g. pmu=off,
vmx=off. As discussed in the thread [1], the best approach is to let users
specify them. How about adding a new option in the configuration file.

Currently this patch only supports this option in qemu,no other vmm.

[1] https://github.com/kata-containers/runtime/pull/2559#issuecomment-603998256

Signed-off-by: Jia He <justin.he@arm.com>
Signed-off-by: Peng Tao <bergwolf@hyper.sh>

src/runtime/cli/config/configuration-qemu-virtiofs.toml.in

@@ -37,6 +37,11 @@ firmware = "@FIRMWAREPATH@"
 # For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
 machine_accelerators="@MACHINEACCELERATORS@"
 
+# CPU features
+# comma-separated list of cpu features to pass to the cpu
+# For example, `cpu_features = "pmu=off,vmx=off"
+cpu_features="@CPUFEATURES@"
+
 # Default number of vCPUs per SB/VM:
 # unspecified or 0                --> will be set to @DEFVCPUS@
 # < 0                             --> will be set to the actual number of physical cores

src/runtime/cli/config/configuration-qemu.toml.in

@@ -38,6 +38,11 @@ firmware = "@FIRMWAREPATH@"
 # For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
 machine_accelerators="@MACHINEACCELERATORS@"
 
+# CPU features
+# comma-separated list of cpu features to pass to the cpu
+# For example, `cpu_features = "pmu=off,vmx=off"
+cpu_features="@CPUFEATURES@"
+
 # Default number of vCPUs per SB/VM:
 # unspecified or 0                --> will be set to @DEFVCPUS@
 # < 0                             --> will be set to the actual number of physical cores

src/runtime/pkg/katautils/config-settings.go.in

@@ -16,6 +16,7 @@ var defaultKernelPath = "/usr/share/kata-containers/vmlinuz.container"
 var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img"
 var defaultFirmwarePath = ""
 var defaultMachineAccelerators = ""
+var defaultCPUFeatures = ""
 var defaultShimPath = "/usr/libexec/kata-containers/kata-shim"
 var systemdUnitName = "kata-containers.target"
 

src/runtime/pkg/katautils/config.go

@@ -93,6 +93,7 @@ type hypervisor struct {
 	Image                   string   `toml:"image"`
 	Firmware                string   `toml:"firmware"`
 	MachineAccelerators     string   `toml:"machine_accelerators"`
+	CPUFeatures             string   `toml:"cpu_features"`
 	KernelParams            string   `toml:"kernel_params"`
 	MachineType             string   `toml:"machine_type"`
 	BlockDeviceDriver       string   `toml:"block_device_driver"`
@@ -244,11 +245,9 @@ func (h hypervisor) firmware() (string, error) {
 
 func (h hypervisor) machineAccelerators() string {
 	var machineAccelerators string
-	accelerators := strings.Split(h.MachineAccelerators, ",")
+	for _, accelerator := range strings.Split(h.MachineAccelerators, ",") {
-	acceleratorsLen := len(accelerators)
+		if accelerator != "" {
-	for i := 0; i < acceleratorsLen; i++ {
+			machineAccelerators += strings.TrimSpace(accelerator) + ","
-		if accelerators[i] != "" {
-			machineAccelerators += strings.Trim(accelerators[i], "\r\t\n ") + ","
 		}
 	}
 
@@ -257,6 +256,19 @@ func (h hypervisor) machineAccelerators() string {
 	return machineAccelerators
 }
 
+func (h hypervisor) cpuFeatures() string {
+	var cpuFeatures string
+	for _, feature := range strings.Split(h.CPUFeatures, ",") {
+		if feature != "" {
+			cpuFeatures += strings.TrimSpace(feature) + ","
+		}
+	}
+
+	cpuFeatures = strings.Trim(cpuFeatures, ",")
+
+	return cpuFeatures
+}
+
 func (h hypervisor) kernelParams() string {
 	if h.KernelParams == "" {
 		return defaultKernelParams
@@ -624,6 +636,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 	}
 
 	machineAccelerators := h.machineAccelerators()
+	cpuFeatures := h.cpuFeatures()
 	kernelParams := h.kernelParams()
 	machineType := h.machineType()
 
@@ -677,6 +690,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		ImagePath:               image,
 		FirmwarePath:            firmware,
 		MachineAccelerators:     machineAccelerators,
+		CPUFeatures:             cpuFeatures,
 		KernelParams:            vc.DeserializeParams(strings.Fields(kernelParams)),
 		HypervisorMachineType:   machineType,
 		NumVCPUs:                h.defaultVCPUs(),
@@ -1129,6 +1143,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		InitrdPath:              defaultInitrdPath,
 		FirmwarePath:            defaultFirmwarePath,
 		MachineAccelerators:     defaultMachineAccelerators,
+		CPUFeatures:             defaultCPUFeatures,
 		HypervisorMachineType:   defaultMachineType,
 		NumVCPUs:                defaultVCPUCount,
 		DefaultMaxVCPUs:         defaultMaxVCPUCount,

src/runtime/pkg/katautils/config_test.go

@@ -1604,6 +1604,53 @@ func TestDefaultMachineAccelerators(t *testing.T) {
 	assert.Equal(machineAccelerators, h.machineAccelerators())
 }
 
+func TestDefaultCPUFeatures(t *testing.T) {
+	assert := assert.New(t)
+	cpuFeatures := "abc,123,rgb"
+	h := hypervisor{CPUFeatures: cpuFeatures}
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = ""
+	h.CPUFeatures = cpuFeatures
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc"
+	h.CPUFeatures = cpuFeatures
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc,123"
+	h.CPUFeatures = "abc,,123"
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc,123"
+	h.CPUFeatures = ",,abc,,123,,,"
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc,123"
+	h.CPUFeatures = "abc,,123,,,"
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc"
+	h.CPUFeatures = ",,abc,"
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc"
+	h.CPUFeatures = ", , abc , ,"
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc"
+	h.CPUFeatures = " abc "
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc,123"
+	h.CPUFeatures = ", abc , 123 ,"
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+
+	cpuFeatures = "abc,123"
+	h.CPUFeatures = ",, abc ,,, 123 ,,"
+	assert.Equal(cpuFeatures, h.cpuFeatures())
+}
+
 func TestUpdateRuntimeConfiguration(t *testing.T) {
 	assert := assert.New(t)
 

src/runtime/virtcontainers/documentation/api/1.0/api.md

@@ -132,6 +132,9 @@ type HypervisorConfig struct {
 	// MachineAccelerators are machine specific accelerators
 	MachineAccelerators string
 
+	// CPUFeatures are cpu specific features
+	CPUFeatures string
+
 	// HypervisorPath is the hypervisor executable host path.
 	HypervisorPath string
 

src/runtime/virtcontainers/hypervisor.go

@@ -275,6 +275,9 @@ type HypervisorConfig struct {
 	// MachineAccelerators are machine specific accelerators
 	MachineAccelerators string
 
+	// CPUFeatures are cpu specific features
+	CPUFeatures string
+
 	// HypervisorPath is the hypervisor executable host path.
 	HypervisorPath string
 

src/runtime/virtcontainers/persist.go

@@ -221,6 +221,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
 		InitrdPath:              sconfig.HypervisorConfig.InitrdPath,
 		FirmwarePath:            sconfig.HypervisorConfig.FirmwarePath,
 		MachineAccelerators:     sconfig.HypervisorConfig.MachineAccelerators,
+		CPUFeatures:             sconfig.HypervisorConfig.CPUFeatures,
 		HypervisorPath:          sconfig.HypervisorConfig.HypervisorPath,
 		HypervisorCtlPath:       sconfig.HypervisorConfig.HypervisorCtlPath,
 		JailerPath:              sconfig.HypervisorConfig.JailerPath,
@@ -512,6 +513,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
 		InitrdPath:              hconf.InitrdPath,
 		FirmwarePath:            hconf.FirmwarePath,
 		MachineAccelerators:     hconf.MachineAccelerators,
+		CPUFeatures:             hconf.CPUFeatures,
 		HypervisorPath:          hconf.HypervisorPath,
 		HypervisorCtlPath:       hconf.HypervisorCtlPath,
 		JailerPath:              hconf.JailerPath,

src/runtime/virtcontainers/persist/api/config.go

@@ -54,6 +54,9 @@ type HypervisorConfig struct {
 	// MachineAccelerators are machine specific accelerators
 	MachineAccelerators string
 
+	// CPUFeatures are cpu specific features
+	CPUFeatures string
+
 	// HypervisorPath is the hypervisor executable host path.
 	HypervisorPath string
 

src/runtime/virtcontainers/qemu.go

@@ -556,6 +556,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa
 	}
 
 	cpuModel := q.arch.cpuModel()
+	cpuModel += "," + q.config.CPUFeatures
 
 	firmwarePath, err := q.config.FirmwareAssetPath()
 	if err != nil {