diff --git a/src/runtime/cmd/kata-runtime/kata-env.go b/src/runtime/cmd/kata-runtime/kata-env.go index 45ddfbf3e5..e5e7e87ed6 100644 --- a/src/runtime/cmd/kata-runtime/kata-env.go +++ b/src/runtime/cmd/kata-runtime/kata-env.go @@ -114,8 +114,9 @@ type HypervisorInfo struct { Msize9p uint32 MemorySlots uint32 PCIeRootPort uint32 - ColdPlugVFIO hv.PCIePort PCIeSwitchPort uint32 + HotPlugVFIO hv.PCIePort + ColdPlugVFIO hv.PCIePort HotplugVFIOOnRootBus bool Debug bool } @@ -318,9 +319,11 @@ func getHypervisorInfo(config oci.RuntimeConfig) (HypervisorInfo, error) { EntropySource: config.HypervisorConfig.EntropySource, SharedFS: config.HypervisorConfig.SharedFS, VirtioFSDaemon: config.HypervisorConfig.VirtioFSDaemon, + HotPlugVFIO: config.HypervisorConfig.HotPlugVFIO, ColdPlugVFIO: config.HypervisorConfig.ColdPlugVFIO, HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus, PCIeRootPort: config.HypervisorConfig.PCIeRootPort, + PCIeSwitchPort: config.HypervisorConfig.PCIeSwitchPort, SocketPath: socketPath, }, nil } diff --git a/src/runtime/cmd/kata-runtime/kata-env_test.go b/src/runtime/cmd/kata-runtime/kata-env_test.go index 3760104d0c..d897a0c4ed 100644 --- a/src/runtime/cmd/kata-runtime/kata-env_test.go +++ b/src/runtime/cmd/kata-runtime/kata-env_test.go @@ -75,6 +75,7 @@ func createConfig(configPath string, fileData string) error { } func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeConfig, err error) { + var hotPlugVFIO hv.PCIePort var coldPlugVFIO hv.PCIePort const logPath = "/log/path" hypervisorPath := filepath.Join(prefixDir, "hypervisor") @@ -88,6 +89,8 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC enableIOThreads := true hotplugVFIOOnRootBus := true pcieRootPort := uint32(2) + pcieSwitchPort := uint32(2) + hotPlugVFIO = hv.BridgePort coldPlugVFIO = hv.NoPort disableNewNetNs := false sharedFS := "virtio-9p" @@ -132,8 +135,10 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC BlockDeviceDriver: blockStorageDriver, EnableIOThreads: enableIOThreads, HotplugVFIOOnRootBus: hotplugVFIOOnRootBus, + HotPlugVFIO: hotPlugVFIO, ColdPlugVFIO: coldPlugVFIO, PCIeRootPort: pcieRootPort, + PCIeSwitchPort: pcieSwitchPort, DisableNewNetNs: disableNewNetNs, DefaultVCPUCount: hypConfig.NumVCPUs, DefaultMaxVCPUCount: hypConfig.DefaultMaxVCPUs, @@ -278,6 +283,8 @@ func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo { HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus, PCIeRootPort: config.HypervisorConfig.PCIeRootPort, + PCIeSwitchPort: config.HypervisorConfig.PCIeSwitchPort, + HotPlugVFIO: config.HypervisorConfig.HotPlugVFIO, ColdPlugVFIO: config.HypervisorConfig.ColdPlugVFIO, } diff --git a/src/runtime/pkg/containerd-shim-v2/create_test.go b/src/runtime/pkg/containerd-shim-v2/create_test.go index e3e8e9369e..dc37d72dd5 100644 --- a/src/runtime/pkg/containerd-shim-v2/create_test.go +++ b/src/runtime/pkg/containerd-shim-v2/create_test.go @@ -309,6 +309,7 @@ func TestCreateContainerConfigFail(t *testing.T) { } func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err error) { + var hotPlugVFIO hv.PCIePort var coldPlugVFIO hv.PCIePort if dir == "" { return "", fmt.Errorf("BUG: need directory") @@ -331,9 +332,11 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err enableIOThreads := true hotplugVFIOOnRootBus := true pcieRootPort := uint32(2) + pcieSwitchPort := uint32(3) disableNewNetNs := false sharedFS := "virtio-9p" virtioFSdaemon := path.Join(dir, "virtiofsd") + hotPlugVFIO = hv.BridgePort coldPlugVFIO = hv.RootPort configFileOptions := ktu.RuntimeConfigOptions{ @@ -350,9 +353,11 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err EnableIOThreads: enableIOThreads, HotplugVFIOOnRootBus: hotplugVFIOOnRootBus, PCIeRootPort: pcieRootPort, + PCIeSwitchPort: pcieSwitchPort, DisableNewNetNs: disableNewNetNs, SharedFS: sharedFS, VirtioFSDaemon: virtioFSdaemon, + HotPlugVFIO: hotPlugVFIO, ColdPlugVFIO: coldPlugVFIO, } diff --git a/src/runtime/pkg/hypervisors/hypervisor_state.go b/src/runtime/pkg/hypervisors/hypervisor_state.go index 735ac089b4..2eed91cae3 100644 --- a/src/runtime/pkg/hypervisors/hypervisor_state.go +++ b/src/runtime/pkg/hypervisors/hypervisor_state.go @@ -45,13 +45,13 @@ const ( func (p PCIePort) String() string { switch p { case RootPort: - return "root-port" + fallthrough case SwitchPort: - return "switch-port" + fallthrough case BridgePort: - return "bridge-port" + fallthrough case NoPort: - return "no-port" + return string(p) } return fmt.Sprintf("", string(p)) } @@ -76,7 +76,7 @@ type HypervisorState struct { Pid int PCIeRootPort int PCIeSwitchPort int - ColdPlugVFIO PCIePort HotPlugVFIO PCIePort + ColdPlugVFIO PCIePort HotplugVFIOOnRootBus bool } diff --git a/src/runtime/pkg/katatestutils/utils.go b/src/runtime/pkg/katatestutils/utils.go index 4c8257a40f..ef650ea0dd 100644 --- a/src/runtime/pkg/katatestutils/utils.go +++ b/src/runtime/pkg/katatestutils/utils.go @@ -225,6 +225,8 @@ type RuntimeConfigOptions struct { JaegerPassword string PFlash []string PCIeRootPort uint32 + PCIeSwitchPort uint32 + HotPlugVFIO hv.PCIePort ColdPlugVFIO hv.PCIePort DefaultVCPUCount uint32 DefaultMaxVCPUCount uint32 @@ -319,6 +321,7 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string { enable_iothreads = ` + strconv.FormatBool(config.EnableIOThreads) + ` hotplug_vfio_on_root_bus = ` + strconv.FormatBool(config.HotplugVFIOOnRootBus) + ` pcie_root_port = ` + strconv.FormatUint(uint64(config.PCIeRootPort), 10) + ` + pcie_switch_port = ` + strconv.FormatUint(uint64(config.PCIeSwitchPort), 10) + ` cold_plug_vfio = "` + config.ColdPlugVFIO.String() + `" msize_9p = ` + strconv.FormatUint(uint64(config.DefaultMsize9p), 10) + ` enable_debug = ` + strconv.FormatBool(config.HypervisorDebug) + ` diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in index 0de02ce40e..a6825a8a0d 100644 --- a/src/runtime/pkg/katautils/config-settings.go.in +++ b/src/runtime/pkg/katautils/config-settings.go.in @@ -83,6 +83,7 @@ const defaultDisableNestingChecks bool = false const defaultMsize9p uint32 = 8192 const defaultHotplugVFIOOnRootBus bool = false const defaultPCIeRootPort = 0 +const defaultPCIeSwitchPort = 0 const defaultEntropySource = "/dev/urandom" const defaultGuestHookPath string = "" const defaultVirtioFSCacheMode = "never" @@ -108,6 +109,7 @@ const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock" // Default config file used by stateless systems. var defaultRuntimeConfiguration = "@CONFIG_PATH@" -const defaultColdPlugVFIO = hv.NoPort const defaultHotPlugVFIO = hv.BridgePort -const defaultPCIeSwitchPort = 0 +const defaultColdPlugVFIO = hv.NoPort + + diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index e38526fae9..c0a0372dfb 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -149,7 +149,7 @@ type hypervisor struct { EnableIOThreads bool `toml:"enable_iothreads"` DisableImageNvdimm bool `toml:"disable_image_nvdimm"` HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"` - HotPlugVFIO hv.PCIePort `toml:"hotplug_vfio"` + HotPlugVFIO hv.PCIePort `toml:"hot_plug_vfio"` ColdPlugVFIO hv.PCIePort `toml:"cold_plug_vfio"` DisableVhostNet bool `toml:"disable_vhost_net"` GuestMemoryDumpPaging bool `toml:"guest_memory_dump_paging"` @@ -870,8 +870,8 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { Msize9p: h.msize9p(), DisableImageNvdimm: h.DisableImageNvdimm, HotplugVFIOOnRootBus: h.HotplugVFIOOnRootBus, + HotPlugVFIO: h.hotPlugVFIO(), ColdPlugVFIO: h.coldPlugVFIO(), - HotPlugVFIO: h.HotPlugVFIO, PCIeRootPort: h.PCIeRootPort, PCIeSwitchPort: h.PCIeSwitchPort, DisableVhostNet: h.DisableVhostNet, @@ -1674,11 +1674,15 @@ func checkConfig(config oci.RuntimeConfig) error { return err } + hotPlugVFIO := config.HypervisorConfig.HotPlugVFIO coldPlugVFIO := config.HypervisorConfig.ColdPlugVFIO machineType := config.HypervisorConfig.HypervisorMachineType if err := checkPCIeConfig(coldPlugVFIO, machineType); err != nil { return err } + if err := checkPCIeConfig(hotPlugVFIO, machineType); err != nil { + return err + } return nil } @@ -1692,12 +1696,13 @@ func checkPCIeConfig(vfioPort hv.PCIePort, machineType string) error { if machineType != "q35" { return nil } - if vfioPort == hv.NoPort || vfioPort == hv.RootPort || vfioPort == hv.SwitchPort { + if vfioPort == hv.NoPort || vfioPort == hv.BridgePort || + vfioPort == hv.RootPort || vfioPort == hv.SwitchPort { return nil } - return fmt.Errorf("invalid vfio_port=%s setting, allowed values %s, %s, %s", - vfioPort, hv.NoPort, hv.RootPort, hv.SwitchPort) + return fmt.Errorf("invalid vfio_port=%s setting, allowed values %s, %s, %s, %s", + vfioPort, hv.NoPort, hv.BridgePort, hv.RootPort, hv.SwitchPort) } // checkNetNsConfig performs sanity checks on disable_new_netns config. diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index fb506ba6a4..f30a7a27d8 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -71,6 +71,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf if hypervisor == "" { return config, fmt.Errorf("BUG: need hypervisor") } + var hotPlugVFIO hv.PCIePort var coldPlugVFIO hv.PCIePort hypervisorPath := path.Join(dir, "hypervisor") kernelPath := path.Join(dir, "kernel") @@ -86,6 +87,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf enableIOThreads := true hotplugVFIOOnRootBus := true pcieRootPort := uint32(2) + pcieSwitchPort := uint32(3) + hotPlugVFIO = hv.BridgePort coldPlugVFIO = hv.RootPort disableNewNetNs := false sharedFS := "virtio-9p" @@ -109,6 +112,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf EnableIOThreads: enableIOThreads, HotplugVFIOOnRootBus: hotplugVFIOOnRootBus, PCIeRootPort: pcieRootPort, + PCIeSwitchPort: pcieSwitchPort, + HotPlugVFIO: hotPlugVFIO, ColdPlugVFIO: coldPlugVFIO, DisableNewNetNs: disableNewNetNs, DefaultVCPUCount: defaultVCPUCount, @@ -173,6 +178,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf EnableIOThreads: enableIOThreads, HotplugVFIOOnRootBus: hotplugVFIOOnRootBus, PCIeRootPort: pcieRootPort, + PCIeSwitchPort: pcieSwitchPort, + HotPlugVFIO: hotPlugVFIO, ColdPlugVFIO: coldPlugVFIO, Msize9p: defaultMsize9p, MemSlots: defaultMemSlots, @@ -566,9 +573,9 @@ func TestMinimalRuntimeConfig(t *testing.T) { GuestHookPath: defaultGuestHookPath, VhostUserStorePath: defaultVhostUserStorePath, VirtioFSCache: defaultVirtioFSCacheMode, - HotPlugVFIO: defaultHotPlugVFIO, BlockDeviceAIO: defaultBlockDeviceAIO, DisableGuestSeLinux: defaultDisableGuestSeLinux, + HotPlugVFIO: defaultHotPlugVFIO, ColdPlugVFIO: defaultColdPlugVFIO, } diff --git a/src/runtime/virtcontainers/documentation/api/1.0/api.md b/src/runtime/virtcontainers/documentation/api/1.0/api.md index d3071a86f2..a26e653cee 100644 --- a/src/runtime/virtcontainers/documentation/api/1.0/api.md +++ b/src/runtime/virtcontainers/documentation/api/1.0/api.md @@ -289,11 +289,19 @@ type HypervisorConfig struct { HotplugVFIOOnRootBus bool // PCIeRootPort is used to indicate the number of PCIe Root Port devices - // The PCIe Root Port device is used to hot-plug the PCIe device + // The PCIe Root Port device is used to hot(cold)-plug the PCIe device PCIeRootPort uint32 + // PCIeSwitchPort is used to indicate the number of PCIe Switch Ports + // The PCIe Switch port is used to hot(cold)-plug the PCIe device + PCIeSwitchPort uint32 + + // HotPlugVFIO is used to indicate if devices need to be hotplugged on the + // root port, switch, bridge or no port + HotPlugVFIO hv.PCIePort + // ColdPlugVFIO is used to indicate if devices need to be coldplugged on the - // root port, switch or no port + // root port, switch, bridge or no port ColdPlugVFIO hv.PCIePort // BootToBeTemplate used to indicate if the VM is created to be a template VM diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 73dbc6656b..7d14ab448b 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -516,14 +516,15 @@ type HypervisorConfig struct { // ColdPlugVFIO is used to indicate if devices need to be coldplugged on the // root port, switch or no port ColdPlugVFIO hv.PCIePort - // PCIeSwitchPort is used to indicate the number of PCIe Switch devices - // The PCIe Switch Port device is sued to hot-plug PCIe devices - PCIeSwitchPort uint32 // PCIeRootPort is used to indicate the number of PCIe Root Port devices // The PCIe Root Port device is used to hot-plug the PCIe device PCIeRootPort uint32 + // PCIeSwitchPort is used to indicate the number of PCIe Switch devices + // The PCIe Switch Port device is sued to hot-plug PCIe devices + PCIeSwitchPort uint32 + // NumVCPUs specifies default number of vCPUs for the VM. NumVCPUs uint32 diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index 574270c80c..02cdc83875 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -488,8 +488,10 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { DisableNestingChecks: hconf.DisableNestingChecks, DisableImageNvdimm: hconf.DisableImageNvdimm, HotplugVFIOOnRootBus: hconf.HotplugVFIOOnRootBus, + HotPlugVFIO: hconf.HotPlugVFIO, ColdPlugVFIO: hconf.ColdPlugVFIO, PCIeRootPort: hconf.PCIeRootPort, + PCIeSwitchPort: hconf.PCIeSwitchPort, BootToBeTemplate: hconf.BootToBeTemplate, BootFromTemplate: hconf.BootFromTemplate, DisableVhostNet: hconf.DisableVhostNet, diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go index 6457478c58..8cd7520937 100644 --- a/src/runtime/virtcontainers/persist/api/config.go +++ b/src/runtime/virtcontainers/persist/api/config.go @@ -203,8 +203,12 @@ type HypervisorConfig struct { // root bus instead of a bridge. HotplugVFIOOnRootBus bool + // HotPlugVFIO is used to indicate if devices need to be hotplugged on the + // root, switch, bridge or no-port + HotPlugVFIO hv.PCIePort + // ColdPlugVFIO is used to indicate if devices need to be coldplugged on the - // root port or a switch or no-port + // root, bridge, switch or no-port ColdPlugVFIO hv.PCIePort // BootToBeTemplate used to indicate if the VM is created to be a template VM diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 05476ae85e..06e35d3e39 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -84,6 +84,7 @@ type QemuState struct { PCIeRootPort int PCIeSwitchPort int HotplugVFIOOnRootBus bool + HotplugVFIO hv.PCIePort ColdPlugVFIO hv.PCIePort } @@ -283,7 +284,7 @@ func (q *qemu) setup(ctx context.Context, id string, hypervisorConfig *Hyperviso q.Logger().Debug("Creating UUID") q.state.UUID = uuid.Generate().String() - + q.state.HotPlugVFIO = q.config.HotPlugVFIO q.state.ColdPlugVFIO = q.config.ColdPlugVFIO q.state.HotplugVFIOOnRootBus = q.config.HotplugVFIOOnRootBus q.state.PCIeRootPort = int(q.config.PCIeRootPort)