From b1aa8c8a24827983f450daab4f0fffc8b8571f63 Mon Sep 17 00:00:00 2001 From: Zvonko Kaiser Date: Mon, 8 May 2023 12:59:36 +0000 Subject: [PATCH] gpu: Moved the PCIe configs to drivers The hypervisor_state file was the wrong location for the PCIe Port settings, moved everything under device umbrella, where it can be consumed more easily and we do not get into circular deps. Signed-off-by: Zvonko Kaiser --- src/runtime/cmd/kata-runtime/kata-env.go | 6 +- src/runtime/cmd/kata-runtime/kata-env_test.go | 16 +- .../pkg/containerd-shim-v2/create_test.go | 12 +- src/runtime/pkg/device/config/config.go | 65 ++++++ src/runtime/pkg/device/drivers/utils.go | 2 + src/runtime/pkg/device/drivers/vfio.go | 1 - .../pkg/hypervisors/hypervisor_state.go | 34 +--- src/runtime/pkg/katatestutils/utils.go | 6 +- .../pkg/katautils/config-settings.go.in | 6 +- src/runtime/pkg/katautils/config.go | 187 +++++++++--------- src/runtime/pkg/katautils/config_test.go | 32 +-- src/runtime/virtcontainers/hypervisor.go | 4 +- src/runtime/virtcontainers/kata_agent_test.go | 8 +- src/runtime/virtcontainers/nydusd_test.go | 8 +- .../virtcontainers/persist/api/config.go | 6 +- src/runtime/virtcontainers/qemu.go | 37 ++-- src/runtime/virtcontainers/qemu_arch_base.go | 20 +- src/runtime/virtcontainers/sandbox.go | 10 +- 18 files changed, 249 insertions(+), 211 deletions(-) diff --git a/src/runtime/cmd/kata-runtime/kata-env.go b/src/runtime/cmd/kata-runtime/kata-env.go index e5e7e87ed6..d87e426bf5 100644 --- a/src/runtime/cmd/kata-runtime/kata-env.go +++ b/src/runtime/cmd/kata-runtime/kata-env.go @@ -17,7 +17,7 @@ import ( "github.com/prometheus/procfs" "github.com/urfave/cli" - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci" "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" @@ -115,8 +115,8 @@ type HypervisorInfo struct { MemorySlots uint32 PCIeRootPort uint32 PCIeSwitchPort uint32 - HotPlugVFIO hv.PCIePort - ColdPlugVFIO hv.PCIePort + HotPlugVFIO config.PCIePort + ColdPlugVFIO config.PCIePort HotplugVFIOOnRootBus bool Debug bool } diff --git a/src/runtime/cmd/kata-runtime/kata-env_test.go b/src/runtime/cmd/kata-runtime/kata-env_test.go index d897a0c4ed..4de2fb09ef 100644 --- a/src/runtime/cmd/kata-runtime/kata-env_test.go +++ b/src/runtime/cmd/kata-runtime/kata-env_test.go @@ -19,12 +19,12 @@ import ( "testing" "github.com/BurntSushi/toml" - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" vcUtils "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/urfave/cli" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci" @@ -74,9 +74,9 @@ func createConfig(configPath string, fileData string) error { return nil } -func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeConfig, err error) { - var hotPlugVFIO hv.PCIePort - var coldPlugVFIO hv.PCIePort +func makeRuntimeConfig(prefixDir string) (configFile string, ociConfig oci.RuntimeConfig, err error) { + var hotPlugVFIO config.PCIePort + var coldPlugVFIO config.PCIePort const logPath = "/log/path" hypervisorPath := filepath.Join(prefixDir, "hypervisor") kernelPath := filepath.Join(prefixDir, "kernel") @@ -90,8 +90,8 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC hotplugVFIOOnRootBus := true pcieRootPort := uint32(2) pcieSwitchPort := uint32(2) - hotPlugVFIO = hv.BridgePort - coldPlugVFIO = hv.NoPort + hotPlugVFIO = config.BridgePort + coldPlugVFIO = config.NoPort disableNewNetNs := false sharedFS := "virtio-9p" virtioFSdaemon := filepath.Join(prefixDir, "virtiofsd") @@ -161,12 +161,12 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC return "", oci.RuntimeConfig{}, err } - _, config, err = katautils.LoadConfiguration(configFile, true) + _, ociConfig, err = katautils.LoadConfiguration(configFile, true) if err != nil { return "", oci.RuntimeConfig{}, err } - return configFile, config, nil + return configFile, ociConfig, nil } func getExpectedAgentDetails(config oci.RuntimeConfig) (AgentInfo, error) { diff --git a/src/runtime/pkg/containerd-shim-v2/create_test.go b/src/runtime/pkg/containerd-shim-v2/create_test.go index dc37d72dd5..f95b501b31 100644 --- a/src/runtime/pkg/containerd-shim-v2/create_test.go +++ b/src/runtime/pkg/containerd-shim-v2/create_test.go @@ -20,7 +20,7 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations" @@ -308,9 +308,9 @@ func TestCreateContainerConfigFail(t *testing.T) { assert.Error(err) } -func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err error) { - var hotPlugVFIO hv.PCIePort - var coldPlugVFIO hv.PCIePort +func createAllRuntimeConfigFiles(dir, hypervisor string) (runtimeConfig string, err error) { + var hotPlugVFIO config.PCIePort + var coldPlugVFIO config.PCIePort if dir == "" { return "", fmt.Errorf("BUG: need directory") } @@ -336,8 +336,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err disableNewNetNs := false sharedFS := "virtio-9p" virtioFSdaemon := path.Join(dir, "virtiofsd") - hotPlugVFIO = hv.BridgePort - coldPlugVFIO = hv.RootPort + hotPlugVFIO = config.BridgePort + coldPlugVFIO = config.RootPort configFileOptions := ktu.RuntimeConfigOptions{ Hypervisor: "qemu", diff --git a/src/runtime/pkg/device/config/config.go b/src/runtime/pkg/device/config/config.go index f3a0e879ab..2431fa36f1 100644 --- a/src/runtime/pkg/device/config/config.go +++ b/src/runtime/pkg/device/config/config.go @@ -122,6 +122,68 @@ var SysBusPciDevicesPath = "/sys/bus/pci/devices" var getSysDevPath = getSysDevPathImpl +// PCIePortBusPrefix gives us the correct bus nameing dependeing on the port +// used to hot(cold)-plug the device +type PCIePortBusPrefix string + +const ( + PCIeRootPortPrefix PCIePortBusPrefix = "rp" + PCIeSwitchPortPrefix PCIePortBusPrefix = "sw" + PCIeSwitchUpstreamPortPrefix PCIePortBusPrefix = "swup" + PCIeSwitchhDownstreamPortPrefix PCIePortBusPrefix = "swdp" + PCIBridgePortPrefix PCIePortBusPrefix = "bp" +) + +func (p PCIePortBusPrefix) String() string { + switch p { + case PCIeRootPortPrefix: + fallthrough + case PCIeSwitchPortPrefix: + fallthrough + case PCIeSwitchUpstreamPortPrefix: + fallthrough + case PCIeSwitchhDownstreamPortPrefix: + fallthrough + case PCIBridgePortPrefix: + return string(p) + } + return fmt.Sprintf("", string(p)) +} + +// PCIePort distinguish only between root and switch port +type PCIePort string + +const ( + // RootPort attach VFIO devices to a root-port + RootPort PCIePort = "root-port" + // SwitchPort attach VFIO devices to a switch-port + SwitchPort = "switch-port" + // BridgePort is the default + BridgePort = "bridge-port" + // NoPort is for disabling VFIO hotplug/coldplug + NoPort = "no-port" +) + +func (p PCIePort) String() string { + switch p { + case RootPort: + fallthrough + case SwitchPort: + fallthrough + case BridgePort: + fallthrough + case NoPort: + return string(p) + } + return fmt.Sprintf("", string(p)) +} + +var PCIePortPrefixMapping = map[PCIePort]PCIePortBusPrefix{ + RootPort: PCIeRootPortPrefix, + SwitchPort: PCIeSwitchhDownstreamPortPrefix, + BridgePort: PCIBridgePortPrefix, +} + // DeviceInfo is an embedded type that contains device data common to all types of devices. type DeviceInfo struct { // DriverOptions is specific options for each device driver @@ -167,6 +229,9 @@ type DeviceInfo struct { // ColdPlug specifies whether the device must be cold plugged (true) // or hot plugged (false). ColdPlug bool + + // Specifies the PCIe port type to which the device is attached + Port PCIePort } // BlockDrive represents a block storage drive which may be used in case the storage diff --git a/src/runtime/pkg/device/drivers/utils.go b/src/runtime/pkg/device/drivers/utils.go index f0c2109545..ec1609fe93 100644 --- a/src/runtime/pkg/device/drivers/utils.go +++ b/src/runtime/pkg/device/drivers/utils.go @@ -208,7 +208,9 @@ func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo) ([]*config.VFIODe if isPCIe { vfioPCI.Rank = len(AllPCIeDevs) AllPCIeDevs[deviceBDF] = true + vfioPCI.Bus = fmt.Sprintf("%s%d", config.PCIePortPrefixMapping[device.Port], vfioPCI.Rank) } + vfio = vfioPCI case config.VFIOAPDeviceMediatedType: devices, err := GetAPVFIODevices(deviceSysfsDev) diff --git a/src/runtime/pkg/device/drivers/vfio.go b/src/runtime/pkg/device/drivers/vfio.go index 3604a355d0..ad6ec5cc3d 100644 --- a/src/runtime/pkg/device/drivers/vfio.go +++ b/src/runtime/pkg/device/drivers/vfio.go @@ -28,7 +28,6 @@ const ( vfioRemoveIDPath = "/sys/bus/pci/drivers/vfio-pci/remove_id" iommuGroupPath = "/sys/bus/pci/devices/%s/iommu_group" vfioDevPath = "/dev/vfio/%s" - pcieRootPortPrefix = "rp" vfioAPSysfsDir = "/sys/devices/vfio_ap" ) diff --git a/src/runtime/pkg/hypervisors/hypervisor_state.go b/src/runtime/pkg/hypervisors/hypervisor_state.go index 2eed91cae3..20d6c7d657 100644 --- a/src/runtime/pkg/hypervisors/hypervisor_state.go +++ b/src/runtime/pkg/hypervisors/hypervisor_state.go @@ -5,7 +5,7 @@ package hypervisors -import "fmt" +import "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" // Bridge is a bridge where devices can be hot plugged type Bridge struct { @@ -28,34 +28,6 @@ type CPUDevice struct { ID string } -// PCIePort distinguish only between root and switch port -type PCIePort string - -const ( - // RootPort attach VFIO devices to a root-port - RootPort PCIePort = "root-port" - // SwitchPort attach VFIO devices to a switch-port - SwitchPort = "switch-port" - // BridgePort is the default - BridgePort = "bridge-port" - // NoPort is for disabling VFIO hotplug/coldplug - NoPort = "no-port" -) - -func (p PCIePort) String() string { - switch p { - case RootPort: - fallthrough - case SwitchPort: - fallthrough - case BridgePort: - fallthrough - case NoPort: - return string(p) - } - return fmt.Sprintf("", string(p)) -} - type HypervisorState struct { BlockIndexMap map[int]struct{} // Type of hypervisor, E.g. qemu/firecracker/acrn. @@ -76,7 +48,7 @@ type HypervisorState struct { Pid int PCIeRootPort int PCIeSwitchPort int - HotPlugVFIO PCIePort - ColdPlugVFIO PCIePort + HotPlugVFIO config.PCIePort + ColdPlugVFIO config.PCIePort HotplugVFIOOnRootBus bool } diff --git a/src/runtime/pkg/katatestutils/utils.go b/src/runtime/pkg/katatestutils/utils.go index ef650ea0dd..bd76ae9a8b 100644 --- a/src/runtime/pkg/katatestutils/utils.go +++ b/src/runtime/pkg/katatestutils/utils.go @@ -14,7 +14,7 @@ import ( "strconv" "testing" - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" ) @@ -226,8 +226,8 @@ type RuntimeConfigOptions struct { PFlash []string PCIeRootPort uint32 PCIeSwitchPort uint32 - HotPlugVFIO hv.PCIePort - ColdPlugVFIO hv.PCIePort + HotPlugVFIO config.PCIePort + ColdPlugVFIO config.PCIePort DefaultVCPUCount uint32 DefaultMaxVCPUCount uint32 DefaultMemSize uint32 diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in index 521f7b60fa..f3ce570aa3 100644 --- a/src/runtime/pkg/katautils/config-settings.go.in +++ b/src/runtime/pkg/katautils/config-settings.go.in @@ -10,7 +10,7 @@ package katautils import ( - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" + config "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" ) // name is the name of the runtime @@ -109,7 +109,7 @@ const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock" // Default config file used by stateless systems. var defaultRuntimeConfiguration = "@CONFIG_PATH@" -const defaultHotPlugVFIO = hv.NoPort -const defaultColdPlugVFIO = hv.NoPort +const defaultHotPlugVFIO = config.NoPort +const defaultColdPlugVFIO = config.NoPort diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 75a2469844..d34ede93b6 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -20,7 +20,6 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" @@ -78,90 +77,90 @@ type factory struct { } type hypervisor struct { - Path string `toml:"path"` - JailerPath string `toml:"jailer_path"` - Kernel string `toml:"kernel"` - CtlPath string `toml:"ctlpath"` - Initrd string `toml:"initrd"` - Image string `toml:"image"` - RootfsType string `toml:"rootfs_type"` - Firmware string `toml:"firmware"` - FirmwareVolume string `toml:"firmware_volume"` - MachineAccelerators string `toml:"machine_accelerators"` - CPUFeatures string `toml:"cpu_features"` - KernelParams string `toml:"kernel_params"` - MachineType string `toml:"machine_type"` - BlockDeviceDriver string `toml:"block_device_driver"` - EntropySource string `toml:"entropy_source"` - SharedFS string `toml:"shared_fs"` - VirtioFSDaemon string `toml:"virtio_fs_daemon"` - VirtioFSCache string `toml:"virtio_fs_cache"` - VhostUserStorePath string `toml:"vhost_user_store_path"` - FileBackedMemRootDir string `toml:"file_mem_backend"` - GuestHookPath string `toml:"guest_hook_path"` - GuestMemoryDumpPath string `toml:"guest_memory_dump_path"` - SeccompSandbox string `toml:"seccompsandbox"` - BlockDeviceAIO string `toml:"block_device_aio"` - HypervisorPathList []string `toml:"valid_hypervisor_paths"` - JailerPathList []string `toml:"valid_jailer_paths"` - CtlPathList []string `toml:"valid_ctlpaths"` - VirtioFSDaemonList []string `toml:"valid_virtio_fs_daemon_paths"` - VirtioFSExtraArgs []string `toml:"virtio_fs_extra_args"` - PFlashList []string `toml:"pflashes"` - VhostUserStorePathList []string `toml:"valid_vhost_user_store_paths"` - FileBackedMemRootList []string `toml:"valid_file_mem_backends"` - EntropySourceList []string `toml:"valid_entropy_sources"` - EnableAnnotations []string `toml:"enable_annotations"` - RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"` - TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"` - MemOffset uint64 `toml:"memory_offset"` - DefaultMaxMemorySize uint64 `toml:"default_maxmemory"` - DiskRateLimiterBwMaxRate int64 `toml:"disk_rate_limiter_bw_max_rate"` - DiskRateLimiterBwOneTimeBurst int64 `toml:"disk_rate_limiter_bw_one_time_burst"` - DiskRateLimiterOpsMaxRate int64 `toml:"disk_rate_limiter_ops_max_rate"` - DiskRateLimiterOpsOneTimeBurst int64 `toml:"disk_rate_limiter_ops_one_time_burst"` - NetRateLimiterBwMaxRate int64 `toml:"net_rate_limiter_bw_max_rate"` - NetRateLimiterBwOneTimeBurst int64 `toml:"net_rate_limiter_bw_one_time_burst"` - NetRateLimiterOpsMaxRate int64 `toml:"net_rate_limiter_ops_max_rate"` - NetRateLimiterOpsOneTimeBurst int64 `toml:"net_rate_limiter_ops_one_time_burst"` - VirtioFSCacheSize uint32 `toml:"virtio_fs_cache_size"` - VirtioFSQueueSize uint32 `toml:"virtio_fs_queue_size"` - DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` - MemorySize uint32 `toml:"default_memory"` - MemSlots uint32 `toml:"memory_slots"` - DefaultBridges uint32 `toml:"default_bridges"` - Msize9p uint32 `toml:"msize_9p"` - PCIeSwitchPort uint32 `toml:"pcie_switch_port"` - PCIeRootPort uint32 `toml:"pcie_root_port"` - NumVCPUs int32 `toml:"default_vcpus"` - BlockDeviceCacheSet bool `toml:"block_device_cache_set"` - BlockDeviceCacheDirect bool `toml:"block_device_cache_direct"` - BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"` - EnableVhostUserStore bool `toml:"enable_vhost_user_store"` - VhostUserDeviceReconnect uint32 `toml:"vhost_user_reconnect_timeout_sec"` - DisableBlockDeviceUse bool `toml:"disable_block_device_use"` - MemPrealloc bool `toml:"enable_mem_prealloc"` - HugePages bool `toml:"enable_hugepages"` - VirtioMem bool `toml:"enable_virtio_mem"` - IOMMU bool `toml:"enable_iommu"` - IOMMUPlatform bool `toml:"enable_iommu_platform"` - Debug bool `toml:"enable_debug"` - DisableNestingChecks bool `toml:"disable_nesting_checks"` - EnableIOThreads bool `toml:"enable_iothreads"` - DisableImageNvdimm bool `toml:"disable_image_nvdimm"` - HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"` - HotPlugVFIO hv.PCIePort `toml:"hot_plug_vfio"` - ColdPlugVFIO hv.PCIePort `toml:"cold_plug_vfio"` - DisableVhostNet bool `toml:"disable_vhost_net"` - GuestMemoryDumpPaging bool `toml:"guest_memory_dump_paging"` - ConfidentialGuest bool `toml:"confidential_guest"` - SevSnpGuest bool `toml:"sev_snp_guest"` - GuestSwap bool `toml:"enable_guest_swap"` - Rootless bool `toml:"rootless"` - DisableSeccomp bool `toml:"disable_seccomp"` - DisableSeLinux bool `toml:"disable_selinux"` - DisableGuestSeLinux bool `toml:"disable_guest_selinux"` - LegacySerial bool `toml:"use_legacy_serial"` + Path string `toml:"path"` + JailerPath string `toml:"jailer_path"` + Kernel string `toml:"kernel"` + CtlPath string `toml:"ctlpath"` + Initrd string `toml:"initrd"` + Image string `toml:"image"` + RootfsType string `toml:"rootfs_type"` + Firmware string `toml:"firmware"` + FirmwareVolume string `toml:"firmware_volume"` + MachineAccelerators string `toml:"machine_accelerators"` + CPUFeatures string `toml:"cpu_features"` + KernelParams string `toml:"kernel_params"` + MachineType string `toml:"machine_type"` + BlockDeviceDriver string `toml:"block_device_driver"` + EntropySource string `toml:"entropy_source"` + SharedFS string `toml:"shared_fs"` + VirtioFSDaemon string `toml:"virtio_fs_daemon"` + VirtioFSCache string `toml:"virtio_fs_cache"` + VhostUserStorePath string `toml:"vhost_user_store_path"` + FileBackedMemRootDir string `toml:"file_mem_backend"` + GuestHookPath string `toml:"guest_hook_path"` + GuestMemoryDumpPath string `toml:"guest_memory_dump_path"` + SeccompSandbox string `toml:"seccompsandbox"` + BlockDeviceAIO string `toml:"block_device_aio"` + HypervisorPathList []string `toml:"valid_hypervisor_paths"` + JailerPathList []string `toml:"valid_jailer_paths"` + CtlPathList []string `toml:"valid_ctlpaths"` + VirtioFSDaemonList []string `toml:"valid_virtio_fs_daemon_paths"` + VirtioFSExtraArgs []string `toml:"virtio_fs_extra_args"` + PFlashList []string `toml:"pflashes"` + VhostUserStorePathList []string `toml:"valid_vhost_user_store_paths"` + FileBackedMemRootList []string `toml:"valid_file_mem_backends"` + EntropySourceList []string `toml:"valid_entropy_sources"` + EnableAnnotations []string `toml:"enable_annotations"` + RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"` + TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"` + MemOffset uint64 `toml:"memory_offset"` + DefaultMaxMemorySize uint64 `toml:"default_maxmemory"` + DiskRateLimiterBwMaxRate int64 `toml:"disk_rate_limiter_bw_max_rate"` + DiskRateLimiterBwOneTimeBurst int64 `toml:"disk_rate_limiter_bw_one_time_burst"` + DiskRateLimiterOpsMaxRate int64 `toml:"disk_rate_limiter_ops_max_rate"` + DiskRateLimiterOpsOneTimeBurst int64 `toml:"disk_rate_limiter_ops_one_time_burst"` + NetRateLimiterBwMaxRate int64 `toml:"net_rate_limiter_bw_max_rate"` + NetRateLimiterBwOneTimeBurst int64 `toml:"net_rate_limiter_bw_one_time_burst"` + NetRateLimiterOpsMaxRate int64 `toml:"net_rate_limiter_ops_max_rate"` + NetRateLimiterOpsOneTimeBurst int64 `toml:"net_rate_limiter_ops_one_time_burst"` + VirtioFSCacheSize uint32 `toml:"virtio_fs_cache_size"` + VirtioFSQueueSize uint32 `toml:"virtio_fs_queue_size"` + DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` + MemorySize uint32 `toml:"default_memory"` + MemSlots uint32 `toml:"memory_slots"` + DefaultBridges uint32 `toml:"default_bridges"` + Msize9p uint32 `toml:"msize_9p"` + PCIeSwitchPort uint32 `toml:"pcie_switch_port"` + PCIeRootPort uint32 `toml:"pcie_root_port"` + NumVCPUs int32 `toml:"default_vcpus"` + BlockDeviceCacheSet bool `toml:"block_device_cache_set"` + BlockDeviceCacheDirect bool `toml:"block_device_cache_direct"` + BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"` + EnableVhostUserStore bool `toml:"enable_vhost_user_store"` + VhostUserDeviceReconnect uint32 `toml:"vhost_user_reconnect_timeout_sec"` + DisableBlockDeviceUse bool `toml:"disable_block_device_use"` + MemPrealloc bool `toml:"enable_mem_prealloc"` + HugePages bool `toml:"enable_hugepages"` + VirtioMem bool `toml:"enable_virtio_mem"` + IOMMU bool `toml:"enable_iommu"` + IOMMUPlatform bool `toml:"enable_iommu_platform"` + Debug bool `toml:"enable_debug"` + DisableNestingChecks bool `toml:"disable_nesting_checks"` + EnableIOThreads bool `toml:"enable_iothreads"` + DisableImageNvdimm bool `toml:"disable_image_nvdimm"` + HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"` + HotPlugVFIO config.PCIePort `toml:"hot_plug_vfio"` + ColdPlugVFIO config.PCIePort `toml:"cold_plug_vfio"` + DisableVhostNet bool `toml:"disable_vhost_net"` + GuestMemoryDumpPaging bool `toml:"guest_memory_dump_paging"` + ConfidentialGuest bool `toml:"confidential_guest"` + SevSnpGuest bool `toml:"sev_snp_guest"` + GuestSwap bool `toml:"enable_guest_swap"` + Rootless bool `toml:"rootless"` + DisableSeccomp bool `toml:"disable_seccomp"` + DisableSeLinux bool `toml:"disable_selinux"` + DisableGuestSeLinux bool `toml:"disable_guest_selinux"` + LegacySerial bool `toml:"use_legacy_serial"` } type runtime struct { @@ -289,13 +288,13 @@ func (h hypervisor) firmware() (string, error) { return ResolvePath(p) } -func (h hypervisor) coldPlugVFIO() hv.PCIePort { +func (h hypervisor) coldPlugVFIO() config.PCIePort { if h.ColdPlugVFIO == "" { return defaultColdPlugVFIO } return h.ColdPlugVFIO } -func (h hypervisor) hotPlugVFIO() hv.PCIePort { +func (h hypervisor) hotPlugVFIO() config.PCIePort { if h.HotPlugVFIO == "" { return defaultHotPlugVFIO } @@ -1689,29 +1688,29 @@ func checkConfig(config oci.RuntimeConfig) error { // checkPCIeConfig ensures the PCIe configuration is valid. // Only allow one of the following settings for cold-plug: // no-port, root-port, switch-port -func checkPCIeConfig(coldPlug hv.PCIePort, hotPlug hv.PCIePort, machineType string) error { +func checkPCIeConfig(coldPlug config.PCIePort, hotPlug config.PCIePort, machineType string) error { // Currently only QEMU q35 supports advanced PCIe topologies // firecracker, dragonball do not have right now any PCIe support if machineType != "q35" { return nil } - if coldPlug != hv.NoPort && hotPlug != hv.NoPort { + if coldPlug != config.NoPort && hotPlug != config.NoPort { return fmt.Errorf("invalid hot-plug=%s and cold-plug=%s settings, only one of them can be set", coldPlug, hotPlug) } - var port hv.PCIePort - if coldPlug != hv.NoPort { + var port config.PCIePort + if coldPlug != config.NoPort { port = coldPlug } - if hotPlug != hv.NoPort { + if hotPlug != config.NoPort { port = hotPlug } - if port == hv.NoPort || port == hv.BridgePort || port == hv.RootPort || port == hv.SwitchPort { + if port == config.NoPort || port == config.BridgePort || port == config.RootPort || port == config.SwitchPort { return nil } return fmt.Errorf("invalid vfio_port=%s setting, allowed values %s, %s, %s, %s", - coldPlug, hv.NoPort, hv.BridgePort, hv.RootPort, hv.SwitchPort) + coldPlug, config.NoPort, config.BridgePort, config.RootPort, config.SwitchPort) } // checkNetNsConfig performs sanity checks on disable_new_netns config. diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index f30a7a27d8..2f062bd08a 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -18,8 +18,8 @@ import ( "syscall" "testing" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" @@ -63,16 +63,16 @@ func createConfig(configPath string, fileData string) error { // createAllRuntimeConfigFiles creates all files necessary to call // loadConfiguration(). -func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConfig, err error) { +func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntimeConfig, err error) { if dir == "" { - return config, fmt.Errorf("BUG: need directory") + return testConfig, fmt.Errorf("BUG: need directory") } if hypervisor == "" { - return config, fmt.Errorf("BUG: need hypervisor") + return testConfig, fmt.Errorf("BUG: need hypervisor") } - var hotPlugVFIO hv.PCIePort - var coldPlugVFIO hv.PCIePort + var hotPlugVFIO config.PCIePort + var coldPlugVFIO config.PCIePort hypervisorPath := path.Join(dir, "hypervisor") kernelPath := path.Join(dir, "kernel") kernelParams := "foo=bar xyz" @@ -88,8 +88,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf hotplugVFIOOnRootBus := true pcieRootPort := uint32(2) pcieSwitchPort := uint32(3) - hotPlugVFIO = hv.BridgePort - coldPlugVFIO = hv.RootPort + hotPlugVFIO = config.BridgePort + coldPlugVFIO = config.RootPort disableNewNetNs := false sharedFS := "virtio-9p" virtioFSdaemon := path.Join(dir, "virtiofsd") @@ -139,7 +139,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf configPath := path.Join(dir, "runtime.toml") err = createConfig(configPath, runtimeConfigFileData) if err != nil { - return config, err + return testConfig, err } configPathLink := path.Join(filepath.Dir(configPath), "link-to-configuration.toml") @@ -147,7 +147,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf // create a link to the config file err = syscall.Symlink(configPath, configPathLink) if err != nil { - return config, err + return testConfig, err } files := []string{hypervisorPath, kernelPath, imagePath} @@ -156,7 +156,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf // create the resource (which must be >0 bytes) err := WriteFile(file, "foo", testFileMode) if err != nil { - return config, err + return testConfig, err } } @@ -223,10 +223,10 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf err = SetKernelParams(&runtimeConfig) if err != nil { - return config, err + return testConfig, err } - config = testRuntimeConfig{ + rtimeConfig := testRuntimeConfig{ RuntimeConfig: runtimeConfig, RuntimeConfigFile: configPath, ConfigPath: configPath, @@ -235,7 +235,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf LogPath: logPath, } - return config, nil + return rtimeConfig, nil } // testLoadConfiguration accepts an optional function that can be used @@ -610,7 +610,7 @@ func TestMinimalRuntimeConfig(t *testing.T) { func TestNewQemuHypervisorConfig(t *testing.T) { dir := t.TempDir() - var coldPlugVFIO hv.PCIePort + var coldPlugVFIO config.PCIePort hypervisorPath := path.Join(dir, "hypervisor") kernelPath := path.Join(dir, "kernel") imagePath := path.Join(dir, "image") @@ -619,7 +619,7 @@ func TestNewQemuHypervisorConfig(t *testing.T) { enableIOThreads := true hotplugVFIOOnRootBus := true pcieRootPort := uint32(2) - coldPlugVFIO = hv.RootPort + coldPlugVFIO = config.RootPort orgVHostVSockDevicePath := utils.VHostVSockDevicePath blockDeviceAIO := "io_uring" defer func() { diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 43974adf68..62a8d866f5 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -511,11 +511,11 @@ type HypervisorConfig struct { // HotplugVFIO is used to indicate if devices need to be hotplugged on the // root port or a switch - HotPlugVFIO hv.PCIePort + HotPlugVFIO config.PCIePort // ColdPlugVFIO is used to indicate if devices need to be coldplugged on the // root port, switch or no port - ColdPlugVFIO hv.PCIePort + ColdPlugVFIO config.PCIePort // PCIeRootPort is used to indicate the number of PCIe Root Port devices // The PCIe Root Port device is used to hot-plug the PCIe device diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index 3653674ca9..c7fa059dcb 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -1048,16 +1048,16 @@ func TestKataAgentKernelParams(t *testing.T) { {true, true, 0, []Param{debugParam, traceParam}}, // pipesize - {false, false, 0, []Param{containerPipeSizeParam}}, + {false, false, 2097152, []Param{containerPipeSizeParam}}, // Debug + pipesize - {true, false, 0, []Param{debugParam, containerPipeSizeParam}}, + {true, false, 2097152, []Param{debugParam, containerPipeSizeParam}}, // Tracing + pipesize - {false, true, 0, []Param{traceParam, containerPipeSizeParam}}, + {false, true, 2097152, []Param{traceParam, containerPipeSizeParam}}, // Debug + Tracing + pipesize - {true, true, 0, []Param{debugParam, traceParam, containerPipeSizeParam}}, + {true, true, 2097152, []Param{debugParam, traceParam, containerPipeSizeParam}}, } for i, d := range data { diff --git a/src/runtime/virtcontainers/nydusd_test.go b/src/runtime/virtcontainers/nydusd_test.go index 712113015c..a8ec6dc9b1 100644 --- a/src/runtime/virtcontainers/nydusd_test.go +++ b/src/runtime/virtcontainers/nydusd_test.go @@ -57,13 +57,13 @@ func TestNydusdStart(t *testing.T) { // nolint: govet tests := []struct { - fields fields name string + fields fields wantErr bool }{ - {name: "empty config", fields: fields{}, wantErr: true}, - {name: "directory source path not exist", fields: SourcePathNoExist, wantErr: true}, - {name: "valid config", fields: validConfig, wantErr: false}, + {"empty config", fields{}, true}, + {"directory source path not exist", SourcePathNoExist, true}, + {"valid config", validConfig, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go index 8cd7520937..aeb347127c 100644 --- a/src/runtime/virtcontainers/persist/api/config.go +++ b/src/runtime/virtcontainers/persist/api/config.go @@ -7,7 +7,7 @@ package persistapi import ( - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/opencontainers/runc/libcontainer/configs" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -205,11 +205,11 @@ type HypervisorConfig struct { // HotPlugVFIO is used to indicate if devices need to be hotplugged on the // root, switch, bridge or no-port - HotPlugVFIO hv.PCIePort + HotPlugVFIO config.PCIePort // ColdPlugVFIO is used to indicate if devices need to be coldplugged on the // root, bridge, switch or no-port - ColdPlugVFIO hv.PCIePort + ColdPlugVFIO config.PCIePort // BootToBeTemplate used to indicate if the VM is created to be a template VM BootToBeTemplate bool diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 514cc9ee3c..acd5fcc7e7 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -76,7 +76,7 @@ type qmpChannel struct { // QemuState keeps Qemu's state type QemuState struct { UUID string - HotPlugVFIO hv.PCIePort + HotPlugVFIO config.PCIePort Bridges []types.Bridge HotpluggedVCPUs []hv.CPUDevice HotpluggedMemory int @@ -84,8 +84,8 @@ type QemuState struct { PCIeRootPort int PCIeSwitchPort int HotplugVFIOOnRootBus bool - HotplugVFIO hv.PCIePort - ColdPlugVFIO hv.PCIePort + HotplugVFIO config.PCIePort + ColdPlugVFIO config.PCIePort } // qemu is an Hypervisor interface implementation for the Linux qemu hypervisor. @@ -748,7 +748,7 @@ func (q *qemu) checkBpfEnabled() { // only 16 ports possible. func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig *HypervisorConfig) error { // If no-port set just return no need to add PCIe Root Port or PCIe Switches - if hypervisorConfig.HotPlugVFIO == hv.NoPort && hypervisorConfig.ColdPlugVFIO == hv.NoPort { + if hypervisorConfig.HotPlugVFIO == config.NoPort && hypervisorConfig.ColdPlugVFIO == config.NoPort { return nil } // Add PCIe Root Port or PCIe Switches to the hypervisor @@ -787,11 +787,14 @@ func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig if drivers.IsPCIeDevice(vfioDevice.BDF) { numOfPluggablePorts = numOfPluggablePorts + 1 } - // Reset the Rank, the vfio module is going to assign the correct one - vfioDevice.Rank = -1 + // Reset the Rank and AllPCIeDevsthe vfio module is going + // to assign the correct one in the case of hot-plug + if hypervisorConfig.HotPlugVFIO != config.NoPort { + vfioDevice.Rank = -1 + drivers.AllPCIeDevs = map[string]bool{} + } } } - drivers.AllPCIeDevs = make(map[string]bool) // If number of PCIe root ports > 16 then bail out otherwise we may // use up all slots or IO memory on the root bus and vfio-XXX-pci devices @@ -815,10 +818,10 @@ func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig numOfPluggablePorts = hypervisorConfig.PCIeSwitchPort } - if q.state.HotPlugVFIO == hv.RootPort || q.state.ColdPlugVFIO == hv.RootPort || q.state.HotplugVFIOOnRootBus { + if q.state.HotPlugVFIO == config.RootPort || q.state.ColdPlugVFIO == config.RootPort || q.state.HotplugVFIOOnRootBus { qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, numOfPluggablePorts, memSize32bit, memSize64bit) } - if q.state.HotPlugVFIO == hv.SwitchPort || q.state.ColdPlugVFIO == hv.SwitchPort { + if q.state.HotPlugVFIO == config.SwitchPort || q.state.ColdPlugVFIO == config.SwitchPort { qemuConfig.Devices = q.arch.appendPCIeSwitchPortDevice(qemuConfig.Devices, numOfPluggablePorts, memSize32bit, memSize64bit) } return nil @@ -1629,7 +1632,7 @@ func (q *qemu) hotplugAddVhostUserBlkDevice(ctx context.Context, vAttr *config.V //Since the dev is the first and only one on this bus(root port), it should be 0. addr := "00" - bridgeID := fmt.Sprintf("%s%d", pcieRootPortPrefix, len(drivers.AllPCIeDevs)) + bridgeID := fmt.Sprintf("%s%d", config.PCIeRootPortPrefix, len(drivers.AllPCIeDevs)) drivers.AllPCIeDevs[devID] = true bridgeQomPath := fmt.Sprintf("%s%s", qomPathPrefix, bridgeID) @@ -1824,7 +1827,6 @@ func (q *qemu) hotplugVFIODeviceRootPort(ctx context.Context, device *config.VFI q.state.HotPlugVFIO, q.state.PCIeRootPort, q.state.PCIeSwitchPort) } - device.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, device.Rank) return q.executeVFIODeviceAdd(device) } @@ -1837,7 +1839,6 @@ func (q *qemu) hotplugVFIODeviceSwitchPort(ctx context.Context, device *config.V q.state.HotPlugVFIO, q.state.PCIeRootPort, q.state.PCIeSwitchPort) } - device.Bus = fmt.Sprintf("%s%d", pcieSwitchDownstreamPortPrefix, device.Rank) return q.executeVFIODeviceAdd(device) } @@ -1904,9 +1905,9 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op // In case HotplugVFIOOnRootBus is true, devices are hotplugged on the root bus // for pc machine type instead of bridge. This is useful for devices that require // a large PCI BAR which is a currently a limitation with PCI bridges. - if q.state.HotPlugVFIO == hv.RootPort || q.state.HotplugVFIOOnRootBus { + if q.state.HotPlugVFIO == config.RootPort || q.state.HotplugVFIOOnRootBus { err = q.hotplugVFIODeviceRootPort(ctx, device) - } else if q.state.HotPlugVFIO == hv.SwitchPort { + } else if q.state.HotPlugVFIO == config.SwitchPort { err = q.hotplugVFIODeviceSwitchPort(ctx, device) } else { err = q.hotplugVFIODeviceBridgePort(ctx, device) @@ -2632,7 +2633,7 @@ func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machin for i := uint32(0); i < number; i++ { devices = append(devices, govmmQemu.PCIeRootPortDevice{ - ID: fmt.Sprintf("%s%d", pcieRootPortPrefix, i), + ID: fmt.Sprintf("%s%d", config.PCIeRootPortPrefix, i), Bus: bus, Chassis: chassis, Slot: strconv.FormatUint(uint64(i), 10), @@ -2679,7 +2680,7 @@ func genericAppendPCIeSwitchPort(devices []govmmQemu.Device, number uint32, mach // Using an own ID for the root port, so we do not clash with already // existing root ports adding "s" for switch prefix pcieRootPort := govmmQemu.PCIeRootPortDevice{ - ID: fmt.Sprintf("%s%s%d", pcieSwitchPrefix, pcieRootPortPrefix, 0), + ID: fmt.Sprintf("%s%s%d", config.PCIeSwitchPortPrefix, config.PCIeRootPortPrefix, 0), Bus: defaultBridgeBus, Chassis: "0", Slot: strconv.FormatUint(uint64(0), 10), @@ -2692,7 +2693,7 @@ func genericAppendPCIeSwitchPort(devices []govmmQemu.Device, number uint32, mach devices = append(devices, pcieRootPort) pcieSwitchUpstreamPort := govmmQemu.PCIeSwitchUpstreamPortDevice{ - ID: fmt.Sprintf("%s%d", pcieSwitchUpstreamPortPrefix, 0), + ID: fmt.Sprintf("%s%d", config.PCIeSwitchUpstreamPortPrefix, 0), Bus: pcieRootPort.ID, } devices = append(devices, pcieSwitchUpstreamPort) @@ -2706,7 +2707,7 @@ func genericAppendPCIeSwitchPort(devices []govmmQemu.Device, number uint32, mach for i := uint32(0); i < number; i++ { pcieSwitchDownstreamPort := govmmQemu.PCIeSwitchDownstreamPortDevice{ - ID: fmt.Sprintf("%s%d", pcieSwitchDownstreamPortPrefix, i), + ID: fmt.Sprintf("%s%d", config.PCIeSwitchhDownstreamPortPrefix, i), Bus: pcieSwitchUpstreamPort.ID, Chassis: fmt.Sprintf("%d", nextChassis), Slot: strconv.FormatUint(uint64(i), 10), diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index 9a288d85a7..36a23371fc 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -180,18 +180,14 @@ type qemuArchBase struct { } const ( - defaultCores uint32 = 1 - defaultThreads uint32 = 1 - defaultCPUModel = "host" - defaultBridgeBus = "pcie.0" - defaultPCBridgeBus = "pci.0" - maxDevIDSize = 31 - maxPCIeRootPort = 16 // Limitation from QEMU - maxPCIeSwitchPort = 16 // Limitation from QEMU - pcieRootPortPrefix = "rp" - pcieSwitchPrefix = "sw" - pcieSwitchUpstreamPortPrefix = "swup" - pcieSwitchDownstreamPortPrefix = "swdp" + defaultCores uint32 = 1 + defaultThreads uint32 = 1 + defaultCPUModel = "host" + defaultBridgeBus = "pcie.0" + defaultPCBridgeBus = "pci.0" + maxDevIDSize = 31 + maxPCIeRootPort = 16 // Limitation from QEMU + maxPCIeSwitchPort = 16 // Limitation from QEMU ) // This is the PCI start address assigned to the first bridge that diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index a5e9ca4a5f..9fe88b28ce 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -32,7 +32,6 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers" deviceManager "github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager" - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol" exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental" @@ -616,10 +615,10 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor // If we have a confidential guest we need to cold-plug the PCIe VFIO devices // until we have TDISP/IDE PCIe support. - coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO != hv.NoPort) + coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO != config.NoPort) // Aggregate all the containner devices for hot-plug and use them to dedcue // the correct amount of ports to reserve for the hypervisor. - hotPlugVFIO := (sandboxConfig.HypervisorConfig.HotPlugVFIO != hv.NoPort) + hotPlugVFIO := (sandboxConfig.HypervisorConfig.HotPlugVFIO != config.NoPort) var vfioHotPlugDevices []config.DeviceInfo var vfioColdPlugDevices []config.DeviceInfo @@ -629,9 +628,11 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor isVFIO := deviceManager.IsVFIO(device.ContainerPath) if hotPlugVFIO && isVFIO { vfioHotPlugDevices = append(vfioHotPlugDevices, device) + sandboxConfig.Containers[cnt].DeviceInfos[dev].Port = sandboxConfig.HypervisorConfig.HotPlugVFIO } if coldPlugVFIO && isVFIO { device.ColdPlug = true + device.Port = sandboxConfig.HypervisorConfig.ColdPlugVFIO vfioColdPlugDevices = append(vfioColdPlugDevices, device) // We need to remove the devices marked for cold-plug // otherwise at the container level the kata-agent @@ -639,6 +640,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor infos := sandboxConfig.Containers[cnt].DeviceInfos infos = append(infos[:dev], infos[dev+1:]...) sandboxConfig.Containers[cnt].DeviceInfos = infos + } } } @@ -2015,7 +2017,9 @@ func (s *Sandbox) AppendDevice(ctx context.Context, device api.Device) error { return s.hypervisor.AddDevice(ctx, device.GetDeviceInfo().(*config.VhostUserDeviceAttrs), VhostuserDev) case config.DeviceVFIO: vfioDevs := device.GetDeviceInfo().([]*config.VFIODev) + s.Logger().Info("### vfioDevs: ", vfioDevs) for _, d := range vfioDevs { + s.Logger().Info("### vfioDev: ", d) return s.hypervisor.AddDevice(ctx, *d, VfioDev) } default: