github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-08-15 14:43:51 +00:00
Code Issues Projects Releases Wiki Activity

gpu: Add configuration option for cold-plug VFIO

Browse Source 
Users can set cold-plug="root-port" to cold plug a VFIO device in QEMU

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
This commit is contained in:
Zvonko Kaiser 2023-04-20 11:24:50 +00:00
parent c18ceae109
commit 377ebc2ad1
9 changed files with 104 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/runtime/cmd/kata-runtime/kata-env.go
View File

@ -17,6 +17,7 @@ import (
"github.com/prometheus/procfs" "github.com/prometheus/procfs"
"github.com/urfave/cli" "github.com/urfave/cli"
hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
"github.com/kata-containers/kata-containers/src/runtime/pkg/oci" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
"github.com/kata-containers/kata-containers/src/runtime/pkg/utils" "github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
@ -113,6 +114,7 @@ type HypervisorInfo struct {
Msize9p uint32 Msize9p uint32
MemorySlots uint32 MemorySlots uint32
PCIeRootPort uint32 PCIeRootPort uint32
ColdPlugVFIO hv.PCIePort
HotplugVFIOOnRootBus bool HotplugVFIOOnRootBus bool
Debug bool Debug bool
} }

3
src/runtime/cmd/kata-runtime/kata-env_test.go
View File

@ -24,6 +24,7 @@ import (
specs "github.com/opencontainers/runtime-spec/specs-go" specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/urfave/cli" "github.com/urfave/cli"
hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
"github.com/kata-containers/kata-containers/src/runtime/pkg/oci" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
@ -85,6 +86,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
blockStorageDriver := "virtio-scsi" blockStorageDriver := "virtio-scsi"
enableIOThreads := true enableIOThreads := true
hotplugVFIOOnRootBus := true hotplugVFIOOnRootBus := true
coldPlugVFIO := hv.RootPort
pcieRootPort := uint32(2) pcieRootPort := uint32(2)
disableNewNetNs := false disableNewNetNs := false
sharedFS := "virtio-9p" sharedFS := "virtio-9p"
@ -129,6 +131,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
BlockDeviceDriver: blockStorageDriver, BlockDeviceDriver: blockStorageDriver,
EnableIOThreads: enableIOThreads, EnableIOThreads: enableIOThreads,
HotplugVFIOOnRootBus: hotplugVFIOOnRootBus, HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
ColdPlugVFIO: coldPlugVFIO,
PCIeRootPort: pcieRootPort, PCIeRootPort: pcieRootPort,
DisableNewNetNs: disableNewNetNs, DisableNewNetNs: disableNewNetNs,
DefaultVCPUCount: hypConfig.NumVCPUs, DefaultVCPUCount: hypConfig.NumVCPUs,

2
src/runtime/pkg/katatestutils/utils.go
View File

@ -14,6 +14,7 @@ import (
"strconv" "strconv"
"testing" "testing"
hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
"github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/runtime-spec/specs-go"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
@ -224,6 +225,7 @@ type RuntimeConfigOptions struct {
JaegerPassword string JaegerPassword string
PFlash []string PFlash []string
PCIeRootPort uint32 PCIeRootPort uint32
ColdPlugVFIO hv.PCIePort
DefaultVCPUCount uint32 DefaultVCPUCount uint32
DefaultMaxVCPUCount uint32 DefaultMaxVCPUCount uint32
DefaultMemSize uint32 DefaultMemSize uint32

2
src/runtime/pkg/katautils/config-settings.go.in
View File

@ -103,3 +103,5 @@ const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock"
// Default config file used by stateless systems. // Default config file used by stateless systems.
var defaultRuntimeConfiguration = "@CONFIG_PATH@" var defaultRuntimeConfiguration = "@CONFIG_PATH@"
const defaultColdPlugVFIO = "no-port"

164
src/runtime/pkg/katautils/config.go
View File

@ -20,6 +20,7 @@ import (
"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
"github.com/kata-containers/kata-containers/src/runtime/pkg/oci" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
@ -77,87 +78,88 @@ type factory struct {
} }
type hypervisor struct { type hypervisor struct {
Path string `toml:"path"` Path string `toml:"path"`
JailerPath string `toml:"jailer_path"` JailerPath string `toml:"jailer_path"`
Kernel string `toml:"kernel"` Kernel string `toml:"kernel"`
CtlPath string `toml:"ctlpath"` CtlPath string `toml:"ctlpath"`
Initrd string `toml:"initrd"` Initrd string `toml:"initrd"`
Image string `toml:"image"` Image string `toml:"image"`
RootfsType string `toml:"rootfs_type"` RootfsType string `toml:"rootfs_type"`
Firmware string `toml:"firmware"` Firmware string `toml:"firmware"`
FirmwareVolume string `toml:"firmware_volume"` FirmwareVolume string `toml:"firmware_volume"`
MachineAccelerators string `toml:"machine_accelerators"` MachineAccelerators string `toml:"machine_accelerators"`
CPUFeatures string `toml:"cpu_features"` CPUFeatures string `toml:"cpu_features"`
KernelParams string `toml:"kernel_params"` KernelParams string `toml:"kernel_params"`
MachineType string `toml:"machine_type"` MachineType string `toml:"machine_type"`
BlockDeviceDriver string `toml:"block_device_driver"` BlockDeviceDriver string `toml:"block_device_driver"`
EntropySource string `toml:"entropy_source"` EntropySource string `toml:"entropy_source"`
SharedFS string `toml:"shared_fs"` SharedFS string `toml:"shared_fs"`
VirtioFSDaemon string `toml:"virtio_fs_daemon"` VirtioFSDaemon string `toml:"virtio_fs_daemon"`
VirtioFSCache string `toml:"virtio_fs_cache"` VirtioFSCache string `toml:"virtio_fs_cache"`
VhostUserStorePath string `toml:"vhost_user_store_path"` VhostUserStorePath string `toml:"vhost_user_store_path"`
FileBackedMemRootDir string `toml:"file_mem_backend"` FileBackedMemRootDir string `toml:"file_mem_backend"`
GuestHookPath string `toml:"guest_hook_path"` GuestHookPath string `toml:"guest_hook_path"`
GuestMemoryDumpPath string `toml:"guest_memory_dump_path"` GuestMemoryDumpPath string `toml:"guest_memory_dump_path"`
SeccompSandbox string `toml:"seccompsandbox"` SeccompSandbox string `toml:"seccompsandbox"`
BlockDeviceAIO string `toml:"block_device_aio"` BlockDeviceAIO string `toml:"block_device_aio"`
HypervisorPathList []string `toml:"valid_hypervisor_paths"` HypervisorPathList []string `toml:"valid_hypervisor_paths"`
JailerPathList []string `toml:"valid_jailer_paths"` JailerPathList []string `toml:"valid_jailer_paths"`
CtlPathList []string `toml:"valid_ctlpaths"` CtlPathList []string `toml:"valid_ctlpaths"`
VirtioFSDaemonList []string `toml:"valid_virtio_fs_daemon_paths"` VirtioFSDaemonList []string `toml:"valid_virtio_fs_daemon_paths"`
VirtioFSExtraArgs []string `toml:"virtio_fs_extra_args"` VirtioFSExtraArgs []string `toml:"virtio_fs_extra_args"`
PFlashList []string `toml:"pflashes"` PFlashList []string `toml:"pflashes"`
VhostUserStorePathList []string `toml:"valid_vhost_user_store_paths"` VhostUserStorePathList []string `toml:"valid_vhost_user_store_paths"`
FileBackedMemRootList []string `toml:"valid_file_mem_backends"` FileBackedMemRootList []string `toml:"valid_file_mem_backends"`
EntropySourceList []string `toml:"valid_entropy_sources"` EntropySourceList []string `toml:"valid_entropy_sources"`
EnableAnnotations []string `toml:"enable_annotations"` EnableAnnotations []string `toml:"enable_annotations"`
RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"` RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"`
TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"` TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"`
MemOffset uint64 `toml:"memory_offset"` MemOffset uint64 `toml:"memory_offset"`
DefaultMaxMemorySize uint64 `toml:"default_maxmemory"` DefaultMaxMemorySize uint64 `toml:"default_maxmemory"`
DiskRateLimiterBwMaxRate int64 `toml:"disk_rate_limiter_bw_max_rate"` DiskRateLimiterBwMaxRate int64 `toml:"disk_rate_limiter_bw_max_rate"`
DiskRateLimiterBwOneTimeBurst int64 `toml:"disk_rate_limiter_bw_one_time_burst"` DiskRateLimiterBwOneTimeBurst int64 `toml:"disk_rate_limiter_bw_one_time_burst"`
DiskRateLimiterOpsMaxRate int64 `toml:"disk_rate_limiter_ops_max_rate"` DiskRateLimiterOpsMaxRate int64 `toml:"disk_rate_limiter_ops_max_rate"`
DiskRateLimiterOpsOneTimeBurst int64 `toml:"disk_rate_limiter_ops_one_time_burst"` DiskRateLimiterOpsOneTimeBurst int64 `toml:"disk_rate_limiter_ops_one_time_burst"`
NetRateLimiterBwMaxRate int64 `toml:"net_rate_limiter_bw_max_rate"` NetRateLimiterBwMaxRate int64 `toml:"net_rate_limiter_bw_max_rate"`
NetRateLimiterBwOneTimeBurst int64 `toml:"net_rate_limiter_bw_one_time_burst"` NetRateLimiterBwOneTimeBurst int64 `toml:"net_rate_limiter_bw_one_time_burst"`
NetRateLimiterOpsMaxRate int64 `toml:"net_rate_limiter_ops_max_rate"` NetRateLimiterOpsMaxRate int64 `toml:"net_rate_limiter_ops_max_rate"`
NetRateLimiterOpsOneTimeBurst int64 `toml:"net_rate_limiter_ops_one_time_burst"` NetRateLimiterOpsOneTimeBurst int64 `toml:"net_rate_limiter_ops_one_time_burst"`
VirtioFSCacheSize uint32 `toml:"virtio_fs_cache_size"` VirtioFSCacheSize uint32 `toml:"virtio_fs_cache_size"`
VirtioFSQueueSize uint32 `toml:"virtio_fs_queue_size"` VirtioFSQueueSize uint32 `toml:"virtio_fs_queue_size"`
DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"`
MemorySize uint32 `toml:"default_memory"` MemorySize uint32 `toml:"default_memory"`
MemSlots uint32 `toml:"memory_slots"` MemSlots uint32 `toml:"memory_slots"`
DefaultBridges uint32 `toml:"default_bridges"` DefaultBridges uint32 `toml:"default_bridges"`
Msize9p uint32 `toml:"msize_9p"` Msize9p uint32 `toml:"msize_9p"`
PCIeRootPort uint32 `toml:"pcie_root_port"` PCIeRootPort uint32 `toml:"pcie_root_port"`
NumVCPUs int32 `toml:"default_vcpus"` NumVCPUs int32 `toml:"default_vcpus"`
BlockDeviceCacheSet bool `toml:"block_device_cache_set"` BlockDeviceCacheSet bool `toml:"block_device_cache_set"`
BlockDeviceCacheDirect bool `toml:"block_device_cache_direct"` BlockDeviceCacheDirect bool `toml:"block_device_cache_direct"`
BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"` BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"`
EnableVhostUserStore bool `toml:"enable_vhost_user_store"` EnableVhostUserStore bool `toml:"enable_vhost_user_store"`
VhostUserDeviceReconnect uint32 `toml:"vhost_user_reconnect_timeout_sec"` VhostUserDeviceReconnect uint32 `toml:"vhost_user_reconnect_timeout_sec"`
DisableBlockDeviceUse bool `toml:"disable_block_device_use"` DisableBlockDeviceUse bool `toml:"disable_block_device_use"`
MemPrealloc bool `toml:"enable_mem_prealloc"` MemPrealloc bool `toml:"enable_mem_prealloc"`
HugePages bool `toml:"enable_hugepages"` HugePages bool `toml:"enable_hugepages"`
VirtioMem bool `toml:"enable_virtio_mem"` VirtioMem bool `toml:"enable_virtio_mem"`
IOMMU bool `toml:"enable_iommu"` IOMMU bool `toml:"enable_iommu"`
IOMMUPlatform bool `toml:"enable_iommu_platform"` IOMMUPlatform bool `toml:"enable_iommu_platform"`
Debug bool `toml:"enable_debug"` Debug bool `toml:"enable_debug"`
DisableNestingChecks bool `toml:"disable_nesting_checks"` DisableNestingChecks bool `toml:"disable_nesting_checks"`
EnableIOThreads bool `toml:"enable_iothreads"` EnableIOThreads bool `toml:"enable_iothreads"`
DisableImageNvdimm bool `toml:"disable_image_nvdimm"` DisableImageNvdimm bool `toml:"disable_image_nvdimm"`
HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"` HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"`
DisableVhostNet bool `toml:"disable_vhost_net"` ColdPlugVFIO hv.PCIePort `toml:"cold_plug_vfio"`
GuestMemoryDumpPaging bool `toml:"guest_memory_dump_paging"` DisableVhostNet bool `toml:"disable_vhost_net"`
ConfidentialGuest bool `toml:"confidential_guest"` GuestMemoryDumpPaging bool `toml:"guest_memory_dump_paging"`
SevSnpGuest bool `toml:"sev_snp_guest"` ConfidentialGuest bool `toml:"confidential_guest"`
GuestSwap bool `toml:"enable_guest_swap"` SevSnpGuest bool `toml:"sev_snp_guest"`
Rootless bool `toml:"rootless"` GuestSwap bool `toml:"enable_guest_swap"`
DisableSeccomp bool `toml:"disable_seccomp"` Rootless bool `toml:"rootless"`
DisableSeLinux bool `toml:"disable_selinux"` DisableSeccomp bool `toml:"disable_seccomp"`
DisableGuestSeLinux bool `toml:"disable_guest_selinux"` DisableSeLinux bool `toml:"disable_selinux"`
LegacySerial bool `toml:"use_legacy_serial"` DisableGuestSeLinux bool `toml:"disable_guest_selinux"`
LegacySerial bool `toml:"use_legacy_serial"`
} }
type runtime struct { type runtime struct {

4
src/runtime/virtcontainers/hypervisor.go
View File

@ -509,6 +509,10 @@ type HypervisorConfig struct {
// The PCIe Root Port device is used to hot-plug the PCIe device // The PCIe Root Port device is used to hot-plug the PCIe device
PCIeRootPort uint32 PCIeRootPort uint32
// ColdPlugVFIO is used to indicate if devices need to be coldplugged on the
// root port, switch or no port
ColdPlugVFIO hv.PCIePort
// NumVCPUs specifies default number of vCPUs for the VM. // NumVCPUs specifies default number of vCPUs for the VM.
NumVCPUs uint32 NumVCPUs uint32

3
src/runtime/virtcontainers/kata_agent.go
View File

@ -1177,7 +1177,8 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, c *Container) []*gr
case config.VhostUserBlk: case config.VhostUserBlk:
kataDevice = k.appendVhostUserBlkDevice(dev, device, c) kataDevice = k.appendVhostUserBlkDevice(dev, device, c)
case config.DeviceVFIO: case config.DeviceVFIO:
kataDevice = k.appendVfioDevice(dev, device, c) k.Logger().Infof("### ColdPlugging container is not adding any VFIO devices")
//kataDevice = k.appendVfioDevice(dev, device, c)
} }
if kataDevice == nil { if kataDevice == nil {

1
src/runtime/virtcontainers/persist.go
View File

@ -487,6 +487,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
DisableNestingChecks: hconf.DisableNestingChecks, DisableNestingChecks: hconf.DisableNestingChecks,
DisableImageNvdimm: hconf.DisableImageNvdimm, DisableImageNvdimm: hconf.DisableImageNvdimm,
HotplugVFIOOnRootBus: hconf.HotplugVFIOOnRootBus, HotplugVFIOOnRootBus: hconf.HotplugVFIOOnRootBus,
ColdPlugVFIO: hconf.ColdPlugVFIO,
PCIeRootPort: hconf.PCIeRootPort, PCIeRootPort: hconf.PCIeRootPort,
BootToBeTemplate: hconf.BootToBeTemplate, BootToBeTemplate: hconf.BootToBeTemplate,
BootFromTemplate: hconf.BootFromTemplate, BootFromTemplate: hconf.BootFromTemplate,

5
src/runtime/virtcontainers/persist/api/config.go
View File

@ -7,6 +7,7 @@
package persistapi package persistapi
import ( import (
hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
"github.com/opencontainers/runc/libcontainer/configs" "github.com/opencontainers/runc/libcontainer/configs"
specs "github.com/opencontainers/runtime-spec/specs-go" specs "github.com/opencontainers/runtime-spec/specs-go"
) )
@ -198,6 +199,10 @@ type HypervisorConfig struct {
// root bus instead of a bridge. // root bus instead of a bridge.
HotplugVFIOOnRootBus bool HotplugVFIOOnRootBus bool
// ColdPlugVFIO is used to indicate if devices need to be coldlugged on the
// root port or a switch or no-port
ColdPlugVFIO hv.PCIePort
// BootToBeTemplate used to indicate if the VM is created to be a template VM // BootToBeTemplate used to indicate if the VM is created to be a template VM
BootToBeTemplate bool BootToBeTemplate bool