vfio: Remove obsolete HotplugVFIOonRootBus

Removing HotplugVFIOonRootBus which is obsolete with the latest PCI
topology changes, users can set cold_plug_vfio or hot_plug_vfio either
in the configuration.toml or via annotations.

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>

src/runtime/cmd/kata-runtime/kata-env.go

@@ -115,7 +115,6 @@ type HypervisorInfo struct {
 	MemorySlots       uint32
 	HotPlugVFIO       config.PCIePort
 	ColdPlugVFIO      config.PCIePort
-	HotplugVFIOOnRootBus bool
 	Debug             bool
 }
 
@@ -319,7 +318,6 @@ func getHypervisorInfo(config oci.RuntimeConfig) (HypervisorInfo, error) {
 		VirtioFSDaemon:    config.HypervisorConfig.VirtioFSDaemon,
 		HotPlugVFIO:       config.HypervisorConfig.HotPlugVFIO,
 		ColdPlugVFIO:      config.HypervisorConfig.ColdPlugVFIO,
-		HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
 		SocketPath:        socketPath,
 	}, nil
 }

src/runtime/cmd/kata-runtime/kata-env_test.go

@@ -87,7 +87,6 @@ func makeRuntimeConfig(prefixDir string) (configFile string, ociConfig oci.Runti
 	disableBlock := true
 	blockStorageDriver := "virtio-scsi"
 	enableIOThreads := true
-	hotplugVFIOOnRootBus := true
 	hotPlugVFIO = config.BridgePort
 	coldPlugVFIO = config.NoPort
 	disableNewNetNs := false
@@ -132,7 +131,6 @@ func makeRuntimeConfig(prefixDir string) (configFile string, ociConfig oci.Runti
 		DisableBlock:         disableBlock,
 		BlockDeviceDriver:    blockStorageDriver,
 		EnableIOThreads:      enableIOThreads,
-		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
 		HotPlugVFIO:          hotPlugVFIO,
 		ColdPlugVFIO:         coldPlugVFIO,
 		DisableNewNetNs:      disableNewNetNs,
@@ -276,8 +274,6 @@ func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo {
 		EntropySource:     config.HypervisorConfig.EntropySource,
 		SharedFS:          config.HypervisorConfig.SharedFS,
 		VirtioFSDaemon:    config.HypervisorConfig.VirtioFSDaemon,
-
-		HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
 		HotPlugVFIO:       config.HypervisorConfig.HotPlugVFIO,
 		ColdPlugVFIO:      config.HypervisorConfig.ColdPlugVFIO,
 	}

src/runtime/pkg/containerd-shim-v2/create_test.go

@@ -330,7 +330,6 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (runtimeConfig string,
 	disableBlockDevice := true
 	blockDeviceDriver := "virtio-scsi"
 	enableIOThreads := true
-	hotplugVFIOOnRootBus := true
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
 	virtioFSdaemon := path.Join(dir, "virtiofsd")
@@ -349,7 +348,6 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (runtimeConfig string,
 		DisableBlock:      disableBlockDevice,
 		BlockDeviceDriver: blockDeviceDriver,
 		EnableIOThreads:   enableIOThreads,
-		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
 		DisableNewNetNs:   disableNewNetNs,
 		SharedFS:          sharedFS,
 		VirtioFSDaemon:    virtioFSdaemon,

src/runtime/pkg/hypervisors/hypervisor_state.go

@@ -47,5 +47,4 @@ type HypervisorState struct {
 	Pid               int
 	HotPlugVFIO       config.PCIePort
 	ColdPlugVFIO      config.PCIePort
-	HotplugVFIOOnRootBus bool
 }

src/runtime/pkg/katatestutils/utils.go

@@ -233,7 +233,6 @@ type RuntimeConfigOptions struct {
 	DefaultMsize9p       uint32
 	DisableBlock         bool
 	EnableIOThreads      bool
-	HotplugVFIOOnRootBus bool
 	DisableNewNetNs      bool
 	HypervisorDebug      bool
 	RuntimeDebug         bool
@@ -317,8 +316,8 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
 	default_memory = ` + strconv.FormatUint(uint64(config.DefaultMemSize), 10) + `
 	disable_block_device_use =  ` + strconv.FormatBool(config.DisableBlock) + `
 	enable_iothreads =  ` + strconv.FormatBool(config.EnableIOThreads) + `
-	hotplug_vfio_on_root_bus =  ` + strconv.FormatBool(config.HotplugVFIOOnRootBus) + `
 	cold_plug_vfio =  "` + config.ColdPlugVFIO.String() + `"
+	hot_plug_vfio =   "` + config.HotPlugVFIO.String() + `"
 	msize_9p = ` + strconv.FormatUint(uint64(config.DefaultMsize9p), 10) + `
 	enable_debug = ` + strconv.FormatBool(config.HypervisorDebug) + `
 	guest_hook_path = "` + config.DefaultGuestHookPath + `"

src/runtime/pkg/katautils/config-settings.go.in

@@ -81,7 +81,6 @@ const defaultFileBackedMemRootDir string = ""
 const defaultEnableDebug bool = false
 const defaultDisableNestingChecks bool = false
 const defaultMsize9p uint32 = 8192
-const defaultHotplugVFIOOnRootBus bool = false
 const defaultEntropySource = "/dev/urandom"
 const defaultGuestHookPath string = ""
 const defaultVirtioFSCacheMode = "never"

src/runtime/pkg/katautils/config.go

@@ -146,7 +146,6 @@ type hypervisor struct {
 	DisableNestingChecks           bool            `toml:"disable_nesting_checks"`
 	EnableIOThreads                bool            `toml:"enable_iothreads"`
 	DisableImageNvdimm             bool            `toml:"disable_image_nvdimm"`
-	HotplugVFIOOnRootBus           bool            `toml:"hotplug_vfio_on_root_bus"`
 	HotPlugVFIO                    config.PCIePort `toml:"hot_plug_vfio"`
 	ColdPlugVFIO                   config.PCIePort `toml:"cold_plug_vfio"`
 	DisableVhostNet                bool            `toml:"disable_vhost_net"`
@@ -867,7 +866,6 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		EnableIOThreads:         h.EnableIOThreads,
 		Msize9p:                 h.msize9p(),
 		DisableImageNvdimm:      h.DisableImageNvdimm,
-		HotplugVFIOOnRootBus:    h.HotplugVFIOOnRootBus,
 		HotPlugVFIO:             h.hotPlugVFIO(),
 		ColdPlugVFIO:            h.coldPlugVFIO(),
 		DisableVhostNet:         h.DisableVhostNet,
@@ -1063,7 +1061,6 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		BlockDeviceCacheNoflush:        h.BlockDeviceCacheNoflush,
 		EnableIOThreads:                h.EnableIOThreads,
 		Msize9p:                        h.msize9p(),
-		HotplugVFIOOnRootBus:           h.HotplugVFIOOnRootBus,
 		ColdPlugVFIO:                   h.coldPlugVFIO(),
 		HotPlugVFIO:                    h.hotPlugVFIO(),
 		DisableVhostNet:                true,
@@ -1294,7 +1291,6 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
 		BlockDeviceCacheNoflush:  defaultBlockDeviceCacheNoflush,
 		EnableIOThreads:          defaultEnableIOThreads,
 		Msize9p:                  defaultMsize9p,
-		HotplugVFIOOnRootBus:     defaultHotplugVFIOOnRootBus,
 		ColdPlugVFIO:             defaultColdPlugVFIO,
 		HotPlugVFIO:              defaultHotPlugVFIO,
 		GuestHookPath:            defaultGuestHookPath,
@@ -1682,18 +1678,19 @@ func checkConfig(config oci.RuntimeConfig) error {
 // Only allow one of the following settings for cold-plug:
 // no-port, root-port, switch-port
 func checkPCIeConfig(coldPlug config.PCIePort, hotPlug config.PCIePort, machineType string) error {
-	// Currently only QEMU q35 supports advanced PCIe topologies
+	// Currently only QEMU q35,virt support advanced PCIe topologies
 	// firecracker, dragonball do not have right now any PCIe support
-	if machineType != "q35" {
-		return nil
-	}
-
 	if coldPlug != config.NoPort && hotPlug != config.NoPort {
 		return fmt.Errorf("invalid hot-plug=%s and cold-plug=%s settings, only one of them can be set", coldPlug, hotPlug)
 	}
 	if coldPlug == config.NoPort && hotPlug == config.NoPort {
 		return nil
 	}
+
+	if machineType != "q35" && machineType != "virt" {
+		return nil
+	}
+
 	var port config.PCIePort
 	if coldPlug != config.NoPort {
 		port = coldPlug
@@ -1701,10 +1698,13 @@ func checkPCIeConfig(coldPlug config.PCIePort, hotPlug config.PCIePort, machineT
 	if hotPlug != config.NoPort {
 		port = hotPlug
 	}
-	if port == config.NoPort || port == config.BridgePort || port == config.RootPort || port == config.SwitchPort {
+	if port == config.NoPort {
+		return fmt.Errorf("invalid vfio_port=%s setting, use on of %s, %s, %s",
+			port, config.BridgePort, config.RootPort, config.SwitchPort)
+	}
+	if port == config.BridgePort || port == config.RootPort || port == config.SwitchPort {
 		return nil
 	}
-
 	return fmt.Errorf("invalid vfio_port=%s setting, allowed values %s, %s, %s, %s",
 		coldPlug, config.NoPort, config.BridgePort, config.RootPort, config.SwitchPort)
 }

src/runtime/pkg/katautils/config_test.go

@@ -85,7 +85,6 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntime
 	blockDeviceDriver := "virtio-scsi"
 	blockDeviceAIO := "io_uring"
 	enableIOThreads := true
-	hotplugVFIOOnRootBus := true
 	hotPlugVFIO = config.NoPort
 	coldPlugVFIO = config.BridgePort
 	disableNewNetNs := false
@@ -108,7 +107,6 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntime
 		BlockDeviceDriver:    blockDeviceDriver,
 		BlockDeviceAIO:       blockDeviceAIO,
 		EnableIOThreads:      enableIOThreads,
-		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
 		HotPlugVFIO:          hotPlugVFIO,
 		ColdPlugVFIO:         coldPlugVFIO,
 		DisableNewNetNs:      disableNewNetNs,
@@ -172,7 +170,6 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntime
 		BlockDeviceAIO:        defaultBlockDeviceAIO,
 		DefaultBridges:        defaultBridgesCount,
 		EnableIOThreads:       enableIOThreads,
-		HotplugVFIOOnRootBus:  hotplugVFIOOnRootBus,
 		HotPlugVFIO:           hotPlugVFIO,
 		ColdPlugVFIO:          coldPlugVFIO,
 		Msize9p:               defaultMsize9p,
@@ -611,7 +608,6 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 	machineType := "machineType"
 	disableBlock := true
 	enableIOThreads := true
-	hotplugVFIOOnRootBus := true
 	coldPlugVFIO = config.BridgePort
 	orgVHostVSockDevicePath := utils.VHostVSockDevicePath
 	blockDeviceAIO := "io_uring"
@@ -630,7 +626,6 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 		MachineType:           machineType,
 		DisableBlockDeviceUse: disableBlock,
 		EnableIOThreads:       enableIOThreads,
-		HotplugVFIOOnRootBus:  hotplugVFIOOnRootBus,
 		ColdPlugVFIO:          coldPlugVFIO,
 		RxRateLimiterMaxRate:  rxRateLimiterMaxRate,
 		TxRateLimiterMaxRate:  txRateLimiterMaxRate,
@@ -682,10 +677,6 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 		t.Errorf("Expected value for enable IOThreads  %v, got %v", enableIOThreads, config.EnableIOThreads)
 	}
 
-	if config.HotplugVFIOOnRootBus != hotplugVFIOOnRootBus {
-		t.Errorf("Expected value for HotplugVFIOOnRootBus %v, got %v", hotplugVFIOOnRootBus, config.HotplugVFIOOnRootBus)
-	}
-
 	if config.RxRateLimiterMaxRate != rxRateLimiterMaxRate {
 		t.Errorf("Expected value for rx rate limiter %v, got %v", rxRateLimiterMaxRate, config.RxRateLimiterMaxRate)
 	}
@@ -807,7 +798,6 @@ func TestNewQemuHypervisorConfigImageAndInitrd(t *testing.T) {
 	machineType := "machineType"
 	disableBlock := true
 	enableIOThreads := true
-	hotplugVFIOOnRootBus := true
 
 	hypervisor := hypervisor{
 		Path:                  hypervisorPath,
@@ -817,7 +807,6 @@ func TestNewQemuHypervisorConfigImageAndInitrd(t *testing.T) {
 		MachineType:           machineType,
 		DisableBlockDeviceUse: disableBlock,
 		EnableIOThreads:       enableIOThreads,
-		HotplugVFIOOnRootBus:  hotplugVFIOOnRootBus,
 	}
 
 	_, err := newQemuHypervisorConfig(hypervisor)

src/runtime/pkg/oci/utils.go

@@ -500,12 +500,6 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig,
 		return err
 	}
 
-	if err := newAnnotationConfiguration(ocispec, vcAnnotations.HotplugVFIOOnRootBus).setBool(func(hotplugVFIOOnRootBus bool) {
-		config.HypervisorConfig.HotplugVFIOOnRootBus = hotplugVFIOOnRootBus
-	}); err != nil {
-		return err
-	}
-
 	if err := newAnnotationConfiguration(ocispec, vcAnnotations.UseLegacySerial).setBool(func(useLegacySerial bool) {
 		config.HypervisorConfig.LegacySerial = useLegacySerial
 	}); err != nil {

src/runtime/pkg/oci/utils_test.go

@@ -659,7 +659,6 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	ocispec.Annotations[vcAnnotations.DisableVhostNet] = "true"
 	ocispec.Annotations[vcAnnotations.GuestHookPath] = "/usr/bin/"
 	ocispec.Annotations[vcAnnotations.DisableImageNvdimm] = "true"
-	ocispec.Annotations[vcAnnotations.HotplugVFIOOnRootBus] = "true"
 	ocispec.Annotations[vcAnnotations.ColdPlugVFIO] = config.BridgePort
 	ocispec.Annotations[vcAnnotations.HotPlugVFIO] = config.NoPort
 	ocispec.Annotations[vcAnnotations.IOMMUPlatform] = "true"
@@ -700,7 +699,6 @@ func TestAddHypervisorAnnotations(t *testing.T) {
 	assert.Equal(sbConfig.HypervisorConfig.DisableVhostNet, true)
 	assert.Equal(sbConfig.HypervisorConfig.GuestHookPath, "/usr/bin/")
 	assert.Equal(sbConfig.HypervisorConfig.DisableImageNvdimm, true)
-	assert.Equal(sbConfig.HypervisorConfig.HotplugVFIOOnRootBus, true)
 	assert.Equal(string(sbConfig.HypervisorConfig.ColdPlugVFIO), string(config.BridgePort))
 	assert.Equal(string(sbConfig.HypervisorConfig.HotPlugVFIO), string(config.NoPort))
 	assert.Equal(sbConfig.HypervisorConfig.IOMMUPlatform, true)

src/runtime/virtcontainers/documentation/api/1.0/api.md

@@ -284,10 +284,6 @@ type HypervisorConfig struct {
 	// DisableImageNvdimm is used to disable guest rootfs image nvdimm devices
 	DisableImageNvdimm bool
 
-	// HotplugVFIOOnRootBus is used to indicate if devices need to be hotplugged on the
-	// root bus instead of a bridge.
-	HotplugVFIOOnRootBus bool
-
 	// HotPlugVFIO is used to indicate if devices need to be hotplugged on the
 	// root port, switch, bridge or no port
 	HotPlugVFIO hv.PCIePort

src/runtime/virtcontainers/hypervisor.go

@@ -599,10 +599,6 @@ type HypervisorConfig struct {
 	// DisableImageNvdimm is used to disable guest rootfs image nvdimm devices
 	DisableImageNvdimm bool
 
-	// HotplugVFIOOnRootBus is used to indicate if devices need to be hotplugged on the
-	// root bus instead of a bridge.
-	HotplugVFIOOnRootBus bool
-
 	// GuestMemoryDumpPaging is used to indicate if enable paging
 	// for QEMU dump-guest-memory command
 	GuestMemoryDumpPaging bool

src/runtime/virtcontainers/persist.go

@@ -244,7 +244,6 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
 		FileBackedMemRootList:   sconfig.HypervisorConfig.FileBackedMemRootList,
 		DisableNestingChecks:    sconfig.HypervisorConfig.DisableNestingChecks,
 		DisableImageNvdimm:      sconfig.HypervisorConfig.DisableImageNvdimm,
-		HotplugVFIOOnRootBus:    sconfig.HypervisorConfig.HotplugVFIOOnRootBus,
 		BootToBeTemplate:        sconfig.HypervisorConfig.BootToBeTemplate,
 		BootFromTemplate:        sconfig.HypervisorConfig.BootFromTemplate,
 		DisableVhostNet:         sconfig.HypervisorConfig.DisableVhostNet,
@@ -485,7 +484,6 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
 		FileBackedMemRootList:   hconf.FileBackedMemRootList,
 		DisableNestingChecks:    hconf.DisableNestingChecks,
 		DisableImageNvdimm:      hconf.DisableImageNvdimm,
-		HotplugVFIOOnRootBus:    hconf.HotplugVFIOOnRootBus,
 		HotPlugVFIO:             hconf.HotPlugVFIO,
 		ColdPlugVFIO:            hconf.ColdPlugVFIO,
 		BootToBeTemplate:        hconf.BootToBeTemplate,

src/runtime/virtcontainers/persist/api/config.go

@@ -191,10 +191,6 @@ type HypervisorConfig struct {
 	// DisableImageNvdimm disables nvdimm for guest rootfs image
 	DisableImageNvdimm bool
 
-	// HotplugVFIOOnRootBus is used to indicate if devices need to be hotplugged on the
-	// root bus instead of a bridge.
-	HotplugVFIOOnRootBus bool
-
 	// HotPlugVFIO is used to indicate if devices need to be hotplugged on the
 	// root, switch, bridge or no-port
 	HotPlugVFIO config.PCIePort

src/runtime/virtcontainers/pkg/annotations/annotations.go

@@ -122,10 +122,6 @@ const (
 	// DisableImageNvdimm is a sandbox annotation to specify use of nvdimm device for guest rootfs image.
 	DisableImageNvdimm = kataAnnotHypervisorPrefix + "disable_image_nvdimm"
 
-	// HotplugVFIOOnRootBus is a sandbox annotation used to indicate if devices need to be hotplugged on the
-	// root bus instead of a bridge.
-	HotplugVFIOOnRootBus = kataAnnotHypervisorPrefix + "hotplug_vfio_on_root_bus"
-
 	// ColdPlugVFIO is a sandbox annotation used to indicate if devices need to be coldplugged.
 	ColdPlugVFIO = kataAnnotHypervisorPrefix + "cold_plug_vfio"