runtime-rs: Remove hotplug_vfio_on_root_bus

The `hotplug_vfio_on_root_bus` configuration option is no longer needed
in the runtime-rs.

Remove the option from:
- `DeviceInfo` struct and its `hotplug_vfio_on_root_bus` field
- `PCIeTopology` struct and its construction in `topology.rs`
- `HypervisorInfo` struct and its `hotplug_vfio_on_root_bus` field
- `KATA_ANNO_CFG_HYPERVISOR_HOTPLUG_VFIO_ON_ROOT_BUS` annotation constant
  and its handler in `Annotation::apply()`
- DragonBall validation check in `ConfigPlugin::adjust_config()`
- All runtime-rs TOML config templates (qemu, fc, nvidia GPU variants)
- Test fixture TOML files

Update the documentation having a reference to it.

Signed-off-by: Hyounggyu Choi <Hyounggyu.Choi@ibm.com>

docs/design/kata-vra.md

@@ -138,17 +138,10 @@ user to be in full control of the topology.
 ```toml
 # /etc/kata-containers/configuration.toml
 
-# VFIO devices are hotplugged on a bridge by default.
-# Enable hot-plugging on the root bus. This may be required for devices with
-# a large PCI bar, as this is a current limitation with hot-plugging on
-# a bridge.
-# Default “bridge-port”
-hotplug_vfio = "root-port"
-
 # Before hot plugging a PCIe device, you need to add a pcie_root_port device.
 # Use this parameter when using some large PCI bar devices, such as NVIDIA GPU
 # The value means the number of pcie_root_port
-# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# This value is valid when machine_type is "q35"
 # Default 0
 pcie_root_port = 8
 ```
@@ -284,17 +277,10 @@ PCI Express switch ports to hotplug the devices.
 ```toml
 # /etc/kata-containers/configuration.toml
 
-# VFIO devices are hotplugged on a bridge by default.
-# Enable hot plugging on the root bus. This may be required for devices with
-# a large PCI bar, as this is a current limitation with hot plugging on
-# a bridge.
-# Default “bridge-port”
-hotplug_vfio = "switch-port"
-
-# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
-# Use this parameter when using some large PCI bar devices, such as Nvidia GPU
-# The value means the number of pcie_root_port
-# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# Before hot plugging a PCIe device via a PCI Express switch, you need to add
+# pcie_switch_port devices. Use this parameter when replicating host PCIe switch
+# topology inside the VM. The value means the number of pcie_switch_port
+# This value is valid when machine_type is "q35"
 # Default 0
 pcie_switch_port = 8
 ```

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -84,7 +84,8 @@ Warning: do not enable `virtio_fs_extra_args` in `enable_annotations` unless you
 | `io.katacontainers.config.hypervisor.firmware_volume_hash` | string | container firmware volume SHA-512 hash value |
 | `io.katacontainers.config.hypervisor.firmware_volume` | string | the guest firmware volume that will be passed to the container VM |
 | `io.katacontainers.config.hypervisor.guest_hook_path` | string | the path within the VM that will be used for drop in hooks |
-| `io.katacontainers.config.hypervisor.hotplug_vfio_on_root_bus` | `boolean` | indicate if devices need to be hotplugged on the root bus instead of a bridge|
+| `io.katacontainers.config.hypervisor.cold_plug_vfio` | string | VFIO cold-plug mode; valid values are `no-port`, `bridge-port` (Go runtime only), `root-port`, `switch-port` (Go runtime only) |
+| `io.katacontainers.config.hypervisor.hot_plug_vfio` | string | VFIO hot-plug mode (Go runtime only); valid values are `no-port`, `bridge-port`, `root-port`, `switch-port` |
 | `io.katacontainers.config.hypervisor.hypervisor_hash` | string | container hypervisor binary SHA-512 hash value |
 | `io.katacontainers.config.hypervisor.image_hash` | string | container guest image SHA-512 hash value |
 | `io.katacontainers.config.hypervisor.image` | string | the guest image that will run in the container VM |

docs/use-cases/NVIDIA-GPU-passthrough-and-Kata.md

@@ -100,28 +100,42 @@ instructions](../install/README.md) to install the latest version of Kata.
 To use large BARs devices (for example, NVIDIA Tesla P100), you need Kata
 version 1.11.0 or above.
 
-The following configuration in the Kata `configuration.toml` file as shown below
-can work:
+The following configuration in the Kata `configuration.toml` file can work.
 
-Hotplug for PCI devices with small BARs by `acpi_pcihp` (Linux's ACPI PCI
-Hotplug driver):
+For the Go runtime, hotplug for PCI devices with small BARs by `acpi_pcihp`
+(Linux's ACPI PCI Hotplug driver):
 
 ```sh
 machine_type = "q35"
 
-hotplug_vfio_on_root_bus = false
+hot_plug_vfio = "bridge-port"
 ```
 
-Hotplug for PCIe devices with large BARs by `pciehp` (Linux's PCIe Hotplug
-driver):
+For the Go runtime, hotplug for PCIe devices with large BARs by `pciehp`
+(Linux's PCIe Hotplug driver):
 
 ```sh
 machine_type = "q35"
 
-hotplug_vfio_on_root_bus = true
+hot_plug_vfio = "root-port"
 pcie_root_port = 1
 ```
 
+For `runtime-rs`, hotplug for PCIe devices with large BARs by `pciehp`
+requires pre-creating PCIe ports with `pcie_root_port` or `pcie_switch_port`:
+
+```sh
+machine_type = "q35"
+
+pcie_root_port = 1
+```
+
+```sh
+machine_type = "q35"
+
+pcie_switch_port = 1
+```
+
 ### Build guest kernel with GPU support
 
 The default guest kernel installed with Kata Containers does not provide GPU

src/libs/kata-types/src/annotations/mod.rs

@@ -191,10 +191,6 @@ pub const KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MAX_VCPUS: &str =
     "io.katacontainers.config.hypervisor.default_max_vcpus";
 
 // Hypervisor Device related annotations
-/// A sandbox annotation used to indicate if devices need to be hotplugged on the root bus instead
-/// of a bridge.
-pub const KATA_ANNO_CFG_HYPERVISOR_HOTPLUG_VFIO_ON_ROOT_BUS: &str =
-    "io.katacontainers.config.hypervisor.hotplug_vfio_on_root_bus";
 /// PCIeRootPort is used to indicate the number of PCIe Root Port devices
 pub const KATA_ANNO_CFG_HYPERVISOR_PCIE_ROOT_PORT: &str =
     "io.katacontainers.config.hypervisor.pcie_root_port";
@@ -718,16 +714,6 @@ impl Annotation {
                         }
                     }
                     // Hypervisor Device related annotations
-                    KATA_ANNO_CFG_HYPERVISOR_HOTPLUG_VFIO_ON_ROOT_BUS => {
-                        match self.get_value::<bool>(key) {
-                            Ok(r) => {
-                                hv.device_info.hotplug_vfio_on_root_bus = r.unwrap_or_default();
-                            }
-                            Err(_e) => {
-                                return Err(bool_err);
-                            }
-                        }
-                    }
                     // Limitations documents aligned with runtime-go:
                     // If number of PCIe root ports > 16 then bail out otherwise we may
                     // use up all slots or IO memory on the root bus and vfio-XXX-pci devices

src/libs/kata-types/src/config/hypervisor/dragonball.rs

@@ -157,8 +157,7 @@ impl ConfigPlugin for DragonballConfig {
                     "dragonball hypervisor does not support vIOMMU",
                 ));
             }
-            if db.device_info.hotplug_vfio_on_root_bus
-                || db.device_info.default_bridges > 0
+            if db.device_info.default_bridges > 0
                 || db.device_info.pcie_root_port > 0
                 || db.device_info.pcie_switch_port > 0
             {

src/libs/kata-types/src/config/hypervisor/mod.rs

@@ -810,10 +810,6 @@ pub struct DeviceInfo {
     #[serde(default)]
     pub default_bridges: u32,
 
-    /// Enable hotplugging on root bus for devices with large PCI bars.
-    #[serde(default)]
-    pub hotplug_vfio_on_root_bus: bool,
-
     /// Cold-plug VFIO devices to a PCIe port type.
     ///
     /// Accepted values: `"no-port"` (default, disabled), `"root-port"`.
@@ -824,13 +820,13 @@ pub struct DeviceInfo {
 
     /// Number of PCIe root ports to create during VM creation.
     ///
-    /// Valid when `hotplug_vfio_on_root_bus = true` and `machine_type = "q35"`.
+    /// Valid when `machine_type = "q35"`.
     #[serde(default)]
     pub pcie_root_port: u32,
 
     /// Number of PCIe switch ports to create during VM creation.
     ///
-    /// Valid when `hotplug_vfio_on_root_bus = true` and `machine_type = "q35"`.
+    /// Valid when `machine_type = "q35"`.
     #[serde(default)]
     pub pcie_switch_port: u32,
 

src/libs/kata-types/tests/texture/configuration-anno-0.toml

@@ -47,7 +47,6 @@ enable_iommu_platform = true
 pflashes = ["/proc/mounts"]
 enable_debug = true
 disable_image_nvdimm = true
-hotplug_vfio_on_root_bus = true
 pcie_root_port = 2
 disable_vhost_net = true
 entropy_source= "/dev/urandom"

src/libs/kata-types/tests/texture/configuration-anno-1.toml

@@ -46,7 +46,6 @@ enable_iommu_platform = true
 pflashes = ["/proc/mounts"]
 enable_debug = true
 disable_image_nvdimm = true
-hotplug_vfio_on_root_bus = true
 pcie_root_port = 2
 disable_vhost_net = true
 entropy_source= "/dev/urandom"

src/runtime-rs/config/configuration-qemu-coco-dev-runtime-rs.toml.in

@@ -384,24 +384,17 @@ disable_nesting_checks = false
 # Default is false
 disable_image_nvdimm = false
 
-# VFIO devices are hotplugged on a bridge by default.
-# Enable hotplugging on root bus. This may be required for devices with
-# a large PCI bar, as this is a current limitation with hotplugging on
-# a bridge.
-# Default false
-hotplug_vfio_on_root_bus = false
-
 # Before hot plugging a PCIe device, you need to add a pcie_root_port device.
 # Use this parameter when using some large PCI bar devices, such as Nvidia GPU
 # The value means the number of pcie_root_port
-# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# This value is valid when machine_type is "q35"
 # Default 0
 pcie_root_port = 0
 
 # Before hot plugging a PCIe device onto a switch port, you need add a pcie_switch_port device fist.
 # Use this parameter when using some large PCI bar devices, such as Nvidia GPU
 # The value means how many devices attached onto pcie_switch_port will be created.
-# This value is valid when hotplug_vfio_on_root_bus is true, and machine_type is "q35"
+# This value is valid when machine_type is "q35"
 # Default 0
 pcie_switch_port = 0
 

src/runtime-rs/config/configuration-qemu-nvidia-gpu-runtime-rs.toml.in

@@ -382,13 +382,6 @@ disable_nesting_checks = true
 # Default is false
 disable_image_nvdimm = @DEFDISABLEIMAGENVDIMM_NV@
 
-# VFIO devices are hotplugged on a bridge by default.
-# Enable hotplugging on root bus. This may be required for devices with
-# a large PCI bar, as this is a current limitation with hotplugging on
-# a bridge.
-# Default false
-hotplug_vfio_on_root_bus = false
-
 # Enable cold-plugging of VFIO devices to a PCIe port type.
 # Accepted values: "no-port" (default, disabled), "root-port".
 # When set to "root-port", devices discovered via CDI / Pod Resources
@@ -398,14 +391,14 @@ cold_plug_vfio = "root-port"
 # Before hot plugging a PCIe device, you need to add a pcie_root_port device.
 # Use this parameter when using some large PCI bar devices, such as Nvidia GPU
 # The value means the number of pcie_root_port
-# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# This value is valid when machine_type is "q35"
 # Default 0
 pcie_root_port = @DEFAULTPCIEROOTPORT_NV@
 
 # Before hot plugging a PCIe device onto a switch port, you need add a pcie_switch_port device fist.
 # Use this parameter when using some large PCI bar devices, such as Nvidia GPU
 # The value means how many devices attached onto pcie_switch_port will be created.
-# This value is valid when hotplug_vfio_on_root_bus is true, and machine_type is "q35"
+# This value is valid when machine_type is "q35"
 # Default 0
 pcie_switch_port = 0
 

src/runtime-rs/config/configuration-qemu-nvidia-gpu-snp-runtime-rs.toml.in

@@ -423,13 +423,6 @@ disable_nesting_checks = true
 # Default is false
 disable_image_nvdimm = true
 
-# VFIO devices are hotplugged on a bridge by default.
-# Enable hotplugging on root bus. This may be required for devices with
-# a large PCI bar, as this is a current limitation with hotplugging on
-# a bridge.
-# Default false
-hotplug_vfio_on_root_bus = false
-
 # Enable cold-plugging of VFIO devices to a PCIe port type.
 # Accepted values: "no-port" (default, disabled), "root-port".
 # When set to "root-port", devices discovered via CDI / Pod Resources
@@ -439,14 +432,14 @@ cold_plug_vfio = "root-port"
 # Before hot plugging a PCIe device, you need to add a pcie_root_port device.
 # Use this parameter when using some large PCI bar devices, such as Nvidia GPU
 # The value means the number of pcie_root_port
-# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# This value is valid when machine_type is "q35"
 # Default 0
 pcie_root_port = @DEFAULTPCIEROOTPORT_NV@
 
 # Before hot plugging a PCIe device onto a switch port, you need add a pcie_switch_port device fist.
 # Use this parameter when using some large PCI bar devices, such as Nvidia GPU
 # The value means how many devices attached onto pcie_switch_port will be created.
-# This value is valid when hotplug_vfio_on_root_bus is true, and machine_type is "q35"
+# This value is valid when machine_type is "q35"
 # Default 0
 pcie_switch_port = 0
 

src/runtime-rs/config/configuration-qemu-nvidia-gpu-tdx-runtime-rs.toml.in

@@ -399,13 +399,6 @@ disable_nesting_checks = true
 # Default is false
 disable_image_nvdimm = true
 
-# VFIO devices are hotplugged on a bridge by default.
-# Enable hotplugging on root bus. This may be required for devices with
-# a large PCI bar, as this is a current limitation with hotplugging on
-# a bridge.
-# Default false
-hotplug_vfio_on_root_bus = false
-
 # Enable cold-plugging of VFIO devices to a PCIe port type.
 # Accepted values: "no-port" (default, disabled), "root-port".
 # When set to "root-port", devices discovered via CDI / Pod Resources
@@ -415,14 +408,14 @@ cold_plug_vfio = "root-port"
 # Before hot plugging a PCIe device, you need to add a pcie_root_port device.
 # Use this parameter when using some large PCI bar devices, such as Nvidia GPU
 # The value means the number of pcie_root_port
-# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# This value is valid when machine_type is "q35"
 # Default 0
 pcie_root_port = @DEFAULTPCIEROOTPORT_NV@
 
 # Before hot plugging a PCIe device onto a switch port, you need add a pcie_switch_port device fist.
 # Use this parameter when using some large PCI bar devices, such as Nvidia GPU
 # The value means how many devices attached onto pcie_switch_port will be created.
-# This value is valid when hotplug_vfio_on_root_bus is true, and machine_type is "q35"
+# This value is valid when machine_type is "q35"
 # Default 0
 pcie_switch_port = 0
 

src/runtime-rs/config/configuration-qemu-runtime-rs.toml.in

@@ -374,13 +374,6 @@ disable_nesting_checks = false
 # Default is false
 disable_image_nvdimm = false
 
-# VFIO devices are hotplugged on a bridge by default.
-# Enable hotplugging on root bus. This may be required for devices with
-# a large PCI bar, as this is a current limitation with hotplugging on
-# a bridge.
-# Default false
-hotplug_vfio_on_root_bus = false
-
 # Enable cold-plugging of VFIO devices to a PCIe port type.
 # Accepted values: "no-port" (default, disabled), "root-port".
 # When set to "root-port", devices discovered via CDI / Pod Resources
@@ -390,14 +383,14 @@ cold_plug_vfio = "no-port"
 # Before hot plugging a PCIe device, you need to add a pcie_root_port device.
 # Use this parameter when using some large PCI bar devices, such as Nvidia GPU
 # The value means the number of pcie_root_port
-# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# This value is valid when machine_type is "q35"
 # Default 0
 pcie_root_port = 0
 
 # Before hot plugging a PCIe device onto a switch port, you need add a pcie_switch_port device fist.
 # Use this parameter when using some large PCI bar devices, such as Nvidia GPU
 # The value means how many devices attached onto pcie_switch_port will be created.
-# This value is valid when hotplug_vfio_on_root_bus is true, and machine_type is "q35"
+# This value is valid when machine_type is "q35"
 # Default 0
 pcie_switch_port = 0
 

src/runtime-rs/config/configuration-qemu-se-runtime-rs.toml.in

@@ -358,13 +358,6 @@ disable_nesting_checks = false
 # Default is false
 disable_image_nvdimm = true
 
-# VFIO devices are hotplugged on a bridge by default.
-# Enable hotplugging on root bus. This may be required for devices with
-# a large PCI bar, as this is a current limitation with hotplugging on
-# a bridge.
-# Default false
-hotplug_vfio_on_root_bus = false
-
 # Enable cold-plugging of VFIO devices to a PCIe port type.
 # Accepted values: "no-port" (disabled), "root-port".
 # When set to "root-port", devices discovered via CDI / Pod Resources
@@ -374,7 +367,7 @@ cold_plug_vfio = "root-port"
 # Before hot plugging a PCIe device, you need to add a pcie_root_port device.
 # Use this parameter when using some large PCI bar devices, such as Nvidia GPU
 # The value means the number of pcie_root_port
-# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
+# This value is valid when machine_type is "q35"
 # Default 0
 pcie_root_port = 0
 

src/runtime-rs/config/configuration-rs-fc.toml.in

@@ -203,13 +203,6 @@ enable_debug = false
 # Default false
 disable_nesting_checks = false
 
-# VFIO devices are hotplugged on a bridge by default.
-# Enable hotplugging on root bus. This may be required for devices with
-# a large PCI bar, as this is a current limitation with hotplugging on
-# a bridge.
-# Default false
-hotplug_vfio_on_root_bus = false
-
 #
 # Default entropy source.
 # The path to a host source of entropy (including a real hardware RNG)

src/runtime-rs/crates/hypervisor/src/device/topology.rs

@@ -313,7 +313,6 @@ pub struct PCIeTopology {
     pub cold_plug: bool,
     pub pcie_root_ports: u32,
     pub pcie_switch_ports: u32,
-    pub hotplug_vfio_on_root_bus: bool,
     // pcie_port_devices keeps track of the devices attached to different types of PCI ports.
     pub pcie_port_devices: HashMap<u32, TopologyPortDevice>,
     // device_id -> (bus, bus_slot, port_id)
@@ -353,7 +352,6 @@ impl PCIeTopology {
             cold_plug: true,
             pcie_root_ports: total_rp,
             pcie_switch_ports: total_swp,
-            hotplug_vfio_on_root_bus: topo_config.device_info.hotplug_vfio_on_root_bus,
             pcie_port_devices: HashMap::new(),
             mode,
             reserved_bus: HashMap::new(),

src/runtime-rs/tests/texture/configuration-qemu.toml

@@ -47,7 +47,6 @@ enable_iommu_platform = true
 pflashes = ["/proc/mounts"]
 enable_debug = true
 disable_image_nvdimm = true
-hotplug_vfio_on_root_bus = true
 pcie_root_port = 2
 disable_vhost_net = true
 entropy_source= "/dev/urandom"

src/tools/kata-ctl/src/ops/env_ops.rs

@@ -188,8 +188,6 @@ pub struct HypervisorInfo {
     #[serde(default)]
     pcie_root_port: u32,
     #[serde(default)]
-    hotplug_vfio_on_rootbus: bool,
-    #[serde(default)]
     debug: bool,
     #[serde(default)]
     enable_iommu: bool,
@@ -417,7 +415,6 @@ pub fn get_hypervisor_info(
         virtio_fs_daemon: hypervisor_config.shared_fs.virtio_fs_daemon.to_string(),
         memory_slots: hypervisor_config.memory_info.memory_slots,
         pcie_root_port: hypervisor_config.device_info.pcie_root_port,
-        hotplug_vfio_on_rootbus: hypervisor_config.device_info.hotplug_vfio_on_root_bus,
         debug: hypervisor_config.debug_info.enable_debug,
         enable_iommu: hypervisor_config.device_info.enable_iommu,
         enable_iommu_platform: hypervisor_config.device_info.enable_iommu_platform,