gpu: Add config settings tests for hot-plug

Updated all references and config settings for hot-plug to match
cold-plug

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>

src/runtime/cmd/kata-runtime/kata-env.go

@@ -114,8 +114,9 @@ type HypervisorInfo struct {
 	Msize9p              uint32
 	MemorySlots          uint32
 	PCIeRootPort         uint32
-	ColdPlugVFIO         hv.PCIePort
 	PCIeSwitchPort       uint32
+	HotPlugVFIO          hv.PCIePort
+	ColdPlugVFIO         hv.PCIePort
 	HotplugVFIOOnRootBus bool
 	Debug                bool
 }
@@ -318,9 +319,11 @@ func getHypervisorInfo(config oci.RuntimeConfig) (HypervisorInfo, error) {
 		EntropySource:        config.HypervisorConfig.EntropySource,
 		SharedFS:             config.HypervisorConfig.SharedFS,
 		VirtioFSDaemon:       config.HypervisorConfig.VirtioFSDaemon,
+		HotPlugVFIO:          config.HypervisorConfig.HotPlugVFIO,
 		ColdPlugVFIO:         config.HypervisorConfig.ColdPlugVFIO,
 		HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
 		PCIeRootPort:         config.HypervisorConfig.PCIeRootPort,
+		PCIeSwitchPort:       config.HypervisorConfig.PCIeSwitchPort,
 		SocketPath:           socketPath,
 	}, nil
 }

src/runtime/cmd/kata-runtime/kata-env_test.go

@@ -75,6 +75,7 @@ func createConfig(configPath string, fileData string) error {
 }
 
 func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeConfig, err error) {
+	var hotPlugVFIO hv.PCIePort
 	var coldPlugVFIO hv.PCIePort
 	const logPath = "/log/path"
 	hypervisorPath := filepath.Join(prefixDir, "hypervisor")
@@ -88,6 +89,8 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
 	pcieRootPort := uint32(2)
+	pcieSwitchPort := uint32(2)
+	hotPlugVFIO = hv.BridgePort
 	coldPlugVFIO = hv.NoPort
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
@@ -132,8 +135,10 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 		BlockDeviceDriver:    blockStorageDriver,
 		EnableIOThreads:      enableIOThreads,
 		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
+		HotPlugVFIO:          hotPlugVFIO,
 		ColdPlugVFIO:         coldPlugVFIO,
 		PCIeRootPort:         pcieRootPort,
+		PCIeSwitchPort:       pcieSwitchPort,
 		DisableNewNetNs:      disableNewNetNs,
 		DefaultVCPUCount:     hypConfig.NumVCPUs,
 		DefaultMaxVCPUCount:  hypConfig.DefaultMaxVCPUs,
@@ -278,6 +283,8 @@ func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo {
 
 		HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
 		PCIeRootPort:         config.HypervisorConfig.PCIeRootPort,
+		PCIeSwitchPort:       config.HypervisorConfig.PCIeSwitchPort,
+		HotPlugVFIO:          config.HypervisorConfig.HotPlugVFIO,
 		ColdPlugVFIO:         config.HypervisorConfig.ColdPlugVFIO,
 	}
 

src/runtime/pkg/containerd-shim-v2/create_test.go

@@ -309,6 +309,7 @@ func TestCreateContainerConfigFail(t *testing.T) {
 }
 
 func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err error) {
+	var hotPlugVFIO hv.PCIePort
 	var coldPlugVFIO hv.PCIePort
 	if dir == "" {
 		return "", fmt.Errorf("BUG: need directory")
@@ -331,9 +332,11 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
 	pcieRootPort := uint32(2)
+	pcieSwitchPort := uint32(3)
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
 	virtioFSdaemon := path.Join(dir, "virtiofsd")
+	hotPlugVFIO = hv.BridgePort
 	coldPlugVFIO = hv.RootPort
 
 	configFileOptions := ktu.RuntimeConfigOptions{
@@ -350,9 +353,11 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err
 		EnableIOThreads:      enableIOThreads,
 		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
 		PCIeRootPort:         pcieRootPort,
+		PCIeSwitchPort:       pcieSwitchPort,
 		DisableNewNetNs:      disableNewNetNs,
 		SharedFS:             sharedFS,
 		VirtioFSDaemon:       virtioFSdaemon,
+		HotPlugVFIO:          hotPlugVFIO,
 		ColdPlugVFIO:         coldPlugVFIO,
 	}
 

src/runtime/pkg/hypervisors/hypervisor_state.go

@@ -45,13 +45,13 @@ const (
 func (p PCIePort) String() string {
 	switch p {
 	case RootPort:
-		return "root-port"
+		fallthrough
 	case SwitchPort:
-		return "switch-port"
+		fallthrough
 	case BridgePort:
-		return "bridge-port"
+		fallthrough
 	case NoPort:
-		return "no-port"
+		return string(p)
 	}
 	return fmt.Sprintf("<unknown PCIePort: %s>", string(p))
 }
@@ -76,7 +76,7 @@ type HypervisorState struct {
 	Pid                  int
 	PCIeRootPort         int
 	PCIeSwitchPort       int
-	ColdPlugVFIO         PCIePort
 	HotPlugVFIO          PCIePort
+	ColdPlugVFIO         PCIePort
 	HotplugVFIOOnRootBus bool
 }

src/runtime/pkg/katatestutils/utils.go

@@ -225,6 +225,8 @@ type RuntimeConfigOptions struct {
 	JaegerPassword       string
 	PFlash               []string
 	PCIeRootPort         uint32
+	PCIeSwitchPort       uint32
+	HotPlugVFIO          hv.PCIePort
 	ColdPlugVFIO         hv.PCIePort
 	DefaultVCPUCount     uint32
 	DefaultMaxVCPUCount  uint32
@@ -319,6 +321,7 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
 	enable_iothreads =  ` + strconv.FormatBool(config.EnableIOThreads) + `
 	hotplug_vfio_on_root_bus =  ` + strconv.FormatBool(config.HotplugVFIOOnRootBus) + `
 	pcie_root_port = ` + strconv.FormatUint(uint64(config.PCIeRootPort), 10) + `
+	pcie_switch_port = ` + strconv.FormatUint(uint64(config.PCIeSwitchPort), 10) + `
 	cold_plug_vfio =  "` + config.ColdPlugVFIO.String() + `"
 	msize_9p = ` + strconv.FormatUint(uint64(config.DefaultMsize9p), 10) + `
 	enable_debug = ` + strconv.FormatBool(config.HypervisorDebug) + `

src/runtime/pkg/katautils/config-settings.go.in

@@ -83,6 +83,7 @@ const defaultDisableNestingChecks bool = false
 const defaultMsize9p uint32 = 8192
 const defaultHotplugVFIOOnRootBus bool = false
 const defaultPCIeRootPort = 0
+const defaultPCIeSwitchPort = 0
 const defaultEntropySource = "/dev/urandom"
 const defaultGuestHookPath string = ""
 const defaultVirtioFSCacheMode = "never"
@@ -108,6 +109,7 @@ const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock"
 // Default config file used by stateless systems.
 var defaultRuntimeConfiguration = "@CONFIG_PATH@"
 
-const defaultColdPlugVFIO = hv.NoPort
 const defaultHotPlugVFIO = hv.BridgePort
-const defaultPCIeSwitchPort = 0
+const defaultColdPlugVFIO = hv.NoPort
+
+

src/runtime/pkg/katautils/config.go

@@ -149,7 +149,7 @@ type hypervisor struct {
 	EnableIOThreads                bool        `toml:"enable_iothreads"`
 	DisableImageNvdimm             bool        `toml:"disable_image_nvdimm"`
 	HotplugVFIOOnRootBus           bool        `toml:"hotplug_vfio_on_root_bus"`
-	HotPlugVFIO                    hv.PCIePort `toml:"hotplug_vfio"`
+	HotPlugVFIO                    hv.PCIePort `toml:"hot_plug_vfio"`
 	ColdPlugVFIO                   hv.PCIePort `toml:"cold_plug_vfio"`
 	DisableVhostNet                bool        `toml:"disable_vhost_net"`
 	GuestMemoryDumpPaging          bool        `toml:"guest_memory_dump_paging"`
@@ -870,8 +870,8 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		Msize9p:                 h.msize9p(),
 		DisableImageNvdimm:      h.DisableImageNvdimm,
 		HotplugVFIOOnRootBus:    h.HotplugVFIOOnRootBus,
+		HotPlugVFIO:             h.hotPlugVFIO(),
 		ColdPlugVFIO:            h.coldPlugVFIO(),
-		HotPlugVFIO:             h.HotPlugVFIO,
 		PCIeRootPort:            h.PCIeRootPort,
 		PCIeSwitchPort:          h.PCIeSwitchPort,
 		DisableVhostNet:         h.DisableVhostNet,
@@ -1674,11 +1674,15 @@ func checkConfig(config oci.RuntimeConfig) error {
 		return err
 	}
 
+	hotPlugVFIO := config.HypervisorConfig.HotPlugVFIO
 	coldPlugVFIO := config.HypervisorConfig.ColdPlugVFIO
 	machineType := config.HypervisorConfig.HypervisorMachineType
 	if err := checkPCIeConfig(coldPlugVFIO, machineType); err != nil {
 		return err
 	}
+	if err := checkPCIeConfig(hotPlugVFIO, machineType); err != nil {
+		return err
+	}
 
 	return nil
 }
@@ -1692,12 +1696,13 @@ func checkPCIeConfig(vfioPort hv.PCIePort, machineType string) error {
 	if machineType != "q35" {
 		return nil
 	}
-	if vfioPort == hv.NoPort || vfioPort == hv.RootPort || vfioPort == hv.SwitchPort {
+	if vfioPort == hv.NoPort || vfioPort == hv.BridgePort ||
+		vfioPort == hv.RootPort || vfioPort == hv.SwitchPort {
 		return nil
 	}
 
-	return fmt.Errorf("invalid vfio_port=%s setting, allowed values %s, %s, %s",
+	return fmt.Errorf("invalid vfio_port=%s setting, allowed values %s, %s, %s, %s",
-		vfioPort, hv.NoPort, hv.RootPort, hv.SwitchPort)
+		vfioPort, hv.NoPort, hv.BridgePort, hv.RootPort, hv.SwitchPort)
 }
 
 // checkNetNsConfig performs sanity checks on disable_new_netns config.

src/runtime/pkg/katautils/config_test.go

@@ -71,6 +71,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 	if hypervisor == "" {
 		return config, fmt.Errorf("BUG: need hypervisor")
 	}
+	var hotPlugVFIO hv.PCIePort
 	var coldPlugVFIO hv.PCIePort
 	hypervisorPath := path.Join(dir, "hypervisor")
 	kernelPath := path.Join(dir, "kernel")
@@ -86,6 +87,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
 	pcieRootPort := uint32(2)
+	pcieSwitchPort := uint32(3)
+	hotPlugVFIO = hv.BridgePort
 	coldPlugVFIO = hv.RootPort
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
@@ -109,6 +112,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		EnableIOThreads:      enableIOThreads,
 		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
 		PCIeRootPort:         pcieRootPort,
+		PCIeSwitchPort:       pcieSwitchPort,
+		HotPlugVFIO:          hotPlugVFIO,
 		ColdPlugVFIO:         coldPlugVFIO,
 		DisableNewNetNs:      disableNewNetNs,
 		DefaultVCPUCount:     defaultVCPUCount,
@@ -173,6 +178,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		EnableIOThreads:       enableIOThreads,
 		HotplugVFIOOnRootBus:  hotplugVFIOOnRootBus,
 		PCIeRootPort:          pcieRootPort,
+		PCIeSwitchPort:        pcieSwitchPort,
+		HotPlugVFIO:           hotPlugVFIO,
 		ColdPlugVFIO:          coldPlugVFIO,
 		Msize9p:               defaultMsize9p,
 		MemSlots:              defaultMemSlots,
@@ -566,9 +573,9 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		GuestHookPath:         defaultGuestHookPath,
 		VhostUserStorePath:    defaultVhostUserStorePath,
 		VirtioFSCache:         defaultVirtioFSCacheMode,
-		HotPlugVFIO:           defaultHotPlugVFIO,
 		BlockDeviceAIO:        defaultBlockDeviceAIO,
 		DisableGuestSeLinux:   defaultDisableGuestSeLinux,
+		HotPlugVFIO:           defaultHotPlugVFIO,
 		ColdPlugVFIO:          defaultColdPlugVFIO,
 	}
 

src/runtime/virtcontainers/documentation/api/1.0/api.md

@@ -289,11 +289,19 @@ type HypervisorConfig struct {
 	HotplugVFIOOnRootBus bool
 
 	// PCIeRootPort is used to indicate the number of PCIe Root Port devices
-	// The PCIe Root Port device is used to hot-plug the PCIe device
+	// The PCIe Root Port device is used to hot(cold)-plug the PCIe device
 	PCIeRootPort uint32
 
+	// PCIeSwitchPort is used to indicate the number of PCIe Switch Ports
+	// The PCIe Switch port is used to hot(cold)-plug the PCIe device
+	PCIeSwitchPort uint32
+
+	// HotPlugVFIO is used to indicate if devices need to be hotplugged on the
+	// root port, switch, bridge or no port
+	HotPlugVFIO hv.PCIePort
+
 	// ColdPlugVFIO is used to indicate if devices need to be coldplugged on the
-	// root port, switch or no port
+	// root port, switch, bridge or no port
 	ColdPlugVFIO hv.PCIePort
 
 	// BootToBeTemplate used to indicate if the VM is created to be a template VM

src/runtime/virtcontainers/hypervisor.go

@@ -516,14 +516,15 @@ type HypervisorConfig struct {
 	// ColdPlugVFIO is used to indicate if devices need to be coldplugged on the
 	// root port, switch or no port
 	ColdPlugVFIO hv.PCIePort
-	// PCIeSwitchPort is used to indicate the number of PCIe Switch devices
-	// The PCIe Switch Port device is sued to hot-plug PCIe devices
-	PCIeSwitchPort uint32
 
 	// PCIeRootPort is used to indicate the number of PCIe Root Port devices
 	// The PCIe Root Port device is used to hot-plug the PCIe device
 	PCIeRootPort uint32
 
+	// PCIeSwitchPort is used to indicate the number of PCIe Switch devices
+	// The PCIe Switch Port device is sued to hot-plug PCIe devices
+	PCIeSwitchPort uint32
+
 	// NumVCPUs specifies default number of vCPUs for the VM.
 	NumVCPUs uint32
 

src/runtime/virtcontainers/persist.go

@@ -488,8 +488,10 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
 		DisableNestingChecks:    hconf.DisableNestingChecks,
 		DisableImageNvdimm:      hconf.DisableImageNvdimm,
 		HotplugVFIOOnRootBus:    hconf.HotplugVFIOOnRootBus,
+		HotPlugVFIO:             hconf.HotPlugVFIO,
 		ColdPlugVFIO:            hconf.ColdPlugVFIO,
 		PCIeRootPort:            hconf.PCIeRootPort,
+		PCIeSwitchPort:          hconf.PCIeSwitchPort,
 		BootToBeTemplate:        hconf.BootToBeTemplate,
 		BootFromTemplate:        hconf.BootFromTemplate,
 		DisableVhostNet:         hconf.DisableVhostNet,

src/runtime/virtcontainers/persist/api/config.go

@@ -203,8 +203,12 @@ type HypervisorConfig struct {
 	// root bus instead of a bridge.
 	HotplugVFIOOnRootBus bool
 
+	// HotPlugVFIO is used to indicate if devices need to be hotplugged on the
+	// root, switch, bridge or no-port
+	HotPlugVFIO hv.PCIePort
+
 	// ColdPlugVFIO is used to indicate if devices need to be coldplugged on the
-	// root port or a switch or no-port
+	// root, bridge, switch or no-port
 	ColdPlugVFIO hv.PCIePort
 
 	// BootToBeTemplate used to indicate if the VM is created to be a template VM

src/runtime/virtcontainers/qemu.go

@@ -84,6 +84,7 @@ type QemuState struct {
 	PCIeRootPort         int
 	PCIeSwitchPort       int
 	HotplugVFIOOnRootBus bool
+	HotplugVFIO          hv.PCIePort
 	ColdPlugVFIO         hv.PCIePort
 }
 
@@ -283,7 +284,7 @@ func (q *qemu) setup(ctx context.Context, id string, hypervisorConfig *Hyperviso
 
 		q.Logger().Debug("Creating UUID")
 		q.state.UUID = uuid.Generate().String()
-
+		q.state.HotPlugVFIO = q.config.HotPlugVFIO
 		q.state.ColdPlugVFIO = q.config.ColdPlugVFIO
 		q.state.HotplugVFIOOnRootBus = q.config.HotplugVFIOOnRootBus
 		q.state.PCIeRootPort = int(q.config.PCIeRootPort)