diff --git a/src/runtime/cmd/kata-runtime/kata-env.go b/src/runtime/cmd/kata-runtime/kata-env.go index e5e7e87ed6..d87e426bf5 100644 --- a/src/runtime/cmd/kata-runtime/kata-env.go +++ b/src/runtime/cmd/kata-runtime/kata-env.go @@ -17,7 +17,7 @@ import ( "github.com/prometheus/procfs" "github.com/urfave/cli" - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci" "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" @@ -115,8 +115,8 @@ type HypervisorInfo struct { MemorySlots uint32 PCIeRootPort uint32 PCIeSwitchPort uint32 - HotPlugVFIO hv.PCIePort - ColdPlugVFIO hv.PCIePort + HotPlugVFIO config.PCIePort + ColdPlugVFIO config.PCIePort HotplugVFIOOnRootBus bool Debug bool } diff --git a/src/runtime/cmd/kata-runtime/kata-env_test.go b/src/runtime/cmd/kata-runtime/kata-env_test.go index d897a0c4ed..4de2fb09ef 100644 --- a/src/runtime/cmd/kata-runtime/kata-env_test.go +++ b/src/runtime/cmd/kata-runtime/kata-env_test.go @@ -19,12 +19,12 @@ import ( "testing" "github.com/BurntSushi/toml" - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" vcUtils "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/urfave/cli" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci" @@ -74,9 +74,9 @@ func createConfig(configPath string, fileData string) error { return nil } -func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeConfig, err error) { - var hotPlugVFIO hv.PCIePort - var coldPlugVFIO hv.PCIePort +func makeRuntimeConfig(prefixDir string) (configFile string, ociConfig oci.RuntimeConfig, err error) { + var hotPlugVFIO config.PCIePort + var coldPlugVFIO config.PCIePort const logPath = "/log/path" hypervisorPath := filepath.Join(prefixDir, "hypervisor") kernelPath := filepath.Join(prefixDir, "kernel") @@ -90,8 +90,8 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC hotplugVFIOOnRootBus := true pcieRootPort := uint32(2) pcieSwitchPort := uint32(2) - hotPlugVFIO = hv.BridgePort - coldPlugVFIO = hv.NoPort + hotPlugVFIO = config.BridgePort + coldPlugVFIO = config.NoPort disableNewNetNs := false sharedFS := "virtio-9p" virtioFSdaemon := filepath.Join(prefixDir, "virtiofsd") @@ -161,12 +161,12 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC return "", oci.RuntimeConfig{}, err } - _, config, err = katautils.LoadConfiguration(configFile, true) + _, ociConfig, err = katautils.LoadConfiguration(configFile, true) if err != nil { return "", oci.RuntimeConfig{}, err } - return configFile, config, nil + return configFile, ociConfig, nil } func getExpectedAgentDetails(config oci.RuntimeConfig) (AgentInfo, error) { diff --git a/src/runtime/pkg/containerd-shim-v2/create_test.go b/src/runtime/pkg/containerd-shim-v2/create_test.go index dc37d72dd5..f95b501b31 100644 --- a/src/runtime/pkg/containerd-shim-v2/create_test.go +++ b/src/runtime/pkg/containerd-shim-v2/create_test.go @@ -20,7 +20,7 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations" @@ -308,9 +308,9 @@ func TestCreateContainerConfigFail(t *testing.T) { assert.Error(err) } -func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err error) { - var hotPlugVFIO hv.PCIePort - var coldPlugVFIO hv.PCIePort +func createAllRuntimeConfigFiles(dir, hypervisor string) (runtimeConfig string, err error) { + var hotPlugVFIO config.PCIePort + var coldPlugVFIO config.PCIePort if dir == "" { return "", fmt.Errorf("BUG: need directory") } @@ -336,8 +336,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err disableNewNetNs := false sharedFS := "virtio-9p" virtioFSdaemon := path.Join(dir, "virtiofsd") - hotPlugVFIO = hv.BridgePort - coldPlugVFIO = hv.RootPort + hotPlugVFIO = config.BridgePort + coldPlugVFIO = config.RootPort configFileOptions := ktu.RuntimeConfigOptions{ Hypervisor: "qemu", diff --git a/src/runtime/pkg/device/config/config.go b/src/runtime/pkg/device/config/config.go index f3a0e879ab..2431fa36f1 100644 --- a/src/runtime/pkg/device/config/config.go +++ b/src/runtime/pkg/device/config/config.go @@ -122,6 +122,68 @@ var SysBusPciDevicesPath = "/sys/bus/pci/devices" var getSysDevPath = getSysDevPathImpl +// PCIePortBusPrefix gives us the correct bus nameing dependeing on the port +// used to hot(cold)-plug the device +type PCIePortBusPrefix string + +const ( + PCIeRootPortPrefix PCIePortBusPrefix = "rp" + PCIeSwitchPortPrefix PCIePortBusPrefix = "sw" + PCIeSwitchUpstreamPortPrefix PCIePortBusPrefix = "swup" + PCIeSwitchhDownstreamPortPrefix PCIePortBusPrefix = "swdp" + PCIBridgePortPrefix PCIePortBusPrefix = "bp" +) + +func (p PCIePortBusPrefix) String() string { + switch p { + case PCIeRootPortPrefix: + fallthrough + case PCIeSwitchPortPrefix: + fallthrough + case PCIeSwitchUpstreamPortPrefix: + fallthrough + case PCIeSwitchhDownstreamPortPrefix: + fallthrough + case PCIBridgePortPrefix: + return string(p) + } + return fmt.Sprintf("", string(p)) +} + +// PCIePort distinguish only between root and switch port +type PCIePort string + +const ( + // RootPort attach VFIO devices to a root-port + RootPort PCIePort = "root-port" + // SwitchPort attach VFIO devices to a switch-port + SwitchPort = "switch-port" + // BridgePort is the default + BridgePort = "bridge-port" + // NoPort is for disabling VFIO hotplug/coldplug + NoPort = "no-port" +) + +func (p PCIePort) String() string { + switch p { + case RootPort: + fallthrough + case SwitchPort: + fallthrough + case BridgePort: + fallthrough + case NoPort: + return string(p) + } + return fmt.Sprintf("", string(p)) +} + +var PCIePortPrefixMapping = map[PCIePort]PCIePortBusPrefix{ + RootPort: PCIeRootPortPrefix, + SwitchPort: PCIeSwitchhDownstreamPortPrefix, + BridgePort: PCIBridgePortPrefix, +} + // DeviceInfo is an embedded type that contains device data common to all types of devices. type DeviceInfo struct { // DriverOptions is specific options for each device driver @@ -167,6 +229,9 @@ type DeviceInfo struct { // ColdPlug specifies whether the device must be cold plugged (true) // or hot plugged (false). ColdPlug bool + + // Specifies the PCIe port type to which the device is attached + Port PCIePort } // BlockDrive represents a block storage drive which may be used in case the storage diff --git a/src/runtime/pkg/device/drivers/utils.go b/src/runtime/pkg/device/drivers/utils.go index f0c2109545..ec1609fe93 100644 --- a/src/runtime/pkg/device/drivers/utils.go +++ b/src/runtime/pkg/device/drivers/utils.go @@ -208,7 +208,9 @@ func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo) ([]*config.VFIODe if isPCIe { vfioPCI.Rank = len(AllPCIeDevs) AllPCIeDevs[deviceBDF] = true + vfioPCI.Bus = fmt.Sprintf("%s%d", config.PCIePortPrefixMapping[device.Port], vfioPCI.Rank) } + vfio = vfioPCI case config.VFIOAPDeviceMediatedType: devices, err := GetAPVFIODevices(deviceSysfsDev) diff --git a/src/runtime/pkg/device/drivers/vfio.go b/src/runtime/pkg/device/drivers/vfio.go index 3604a355d0..ad6ec5cc3d 100644 --- a/src/runtime/pkg/device/drivers/vfio.go +++ b/src/runtime/pkg/device/drivers/vfio.go @@ -28,7 +28,6 @@ const ( vfioRemoveIDPath = "/sys/bus/pci/drivers/vfio-pci/remove_id" iommuGroupPath = "/sys/bus/pci/devices/%s/iommu_group" vfioDevPath = "/dev/vfio/%s" - pcieRootPortPrefix = "rp" vfioAPSysfsDir = "/sys/devices/vfio_ap" ) diff --git a/src/runtime/pkg/hypervisors/hypervisor_state.go b/src/runtime/pkg/hypervisors/hypervisor_state.go index 2eed91cae3..20d6c7d657 100644 --- a/src/runtime/pkg/hypervisors/hypervisor_state.go +++ b/src/runtime/pkg/hypervisors/hypervisor_state.go @@ -5,7 +5,7 @@ package hypervisors -import "fmt" +import "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" // Bridge is a bridge where devices can be hot plugged type Bridge struct { @@ -28,34 +28,6 @@ type CPUDevice struct { ID string } -// PCIePort distinguish only between root and switch port -type PCIePort string - -const ( - // RootPort attach VFIO devices to a root-port - RootPort PCIePort = "root-port" - // SwitchPort attach VFIO devices to a switch-port - SwitchPort = "switch-port" - // BridgePort is the default - BridgePort = "bridge-port" - // NoPort is for disabling VFIO hotplug/coldplug - NoPort = "no-port" -) - -func (p PCIePort) String() string { - switch p { - case RootPort: - fallthrough - case SwitchPort: - fallthrough - case BridgePort: - fallthrough - case NoPort: - return string(p) - } - return fmt.Sprintf("", string(p)) -} - type HypervisorState struct { BlockIndexMap map[int]struct{} // Type of hypervisor, E.g. qemu/firecracker/acrn. @@ -76,7 +48,7 @@ type HypervisorState struct { Pid int PCIeRootPort int PCIeSwitchPort int - HotPlugVFIO PCIePort - ColdPlugVFIO PCIePort + HotPlugVFIO config.PCIePort + ColdPlugVFIO config.PCIePort HotplugVFIOOnRootBus bool } diff --git a/src/runtime/pkg/katatestutils/utils.go b/src/runtime/pkg/katatestutils/utils.go index ef650ea0dd..bd76ae9a8b 100644 --- a/src/runtime/pkg/katatestutils/utils.go +++ b/src/runtime/pkg/katatestutils/utils.go @@ -14,7 +14,7 @@ import ( "strconv" "testing" - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" ) @@ -226,8 +226,8 @@ type RuntimeConfigOptions struct { PFlash []string PCIeRootPort uint32 PCIeSwitchPort uint32 - HotPlugVFIO hv.PCIePort - ColdPlugVFIO hv.PCIePort + HotPlugVFIO config.PCIePort + ColdPlugVFIO config.PCIePort DefaultVCPUCount uint32 DefaultMaxVCPUCount uint32 DefaultMemSize uint32 diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in index 521f7b60fa..f3ce570aa3 100644 --- a/src/runtime/pkg/katautils/config-settings.go.in +++ b/src/runtime/pkg/katautils/config-settings.go.in @@ -10,7 +10,7 @@ package katautils import ( - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" + config "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" ) // name is the name of the runtime @@ -109,7 +109,7 @@ const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock" // Default config file used by stateless systems. var defaultRuntimeConfiguration = "@CONFIG_PATH@" -const defaultHotPlugVFIO = hv.NoPort -const defaultColdPlugVFIO = hv.NoPort +const defaultHotPlugVFIO = config.NoPort +const defaultColdPlugVFIO = config.NoPort diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 75a2469844..d34ede93b6 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -20,7 +20,6 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" @@ -78,90 +77,90 @@ type factory struct { } type hypervisor struct { - Path string `toml:"path"` - JailerPath string `toml:"jailer_path"` - Kernel string `toml:"kernel"` - CtlPath string `toml:"ctlpath"` - Initrd string `toml:"initrd"` - Image string `toml:"image"` - RootfsType string `toml:"rootfs_type"` - Firmware string `toml:"firmware"` - FirmwareVolume string `toml:"firmware_volume"` - MachineAccelerators string `toml:"machine_accelerators"` - CPUFeatures string `toml:"cpu_features"` - KernelParams string `toml:"kernel_params"` - MachineType string `toml:"machine_type"` - BlockDeviceDriver string `toml:"block_device_driver"` - EntropySource string `toml:"entropy_source"` - SharedFS string `toml:"shared_fs"` - VirtioFSDaemon string `toml:"virtio_fs_daemon"` - VirtioFSCache string `toml:"virtio_fs_cache"` - VhostUserStorePath string `toml:"vhost_user_store_path"` - FileBackedMemRootDir string `toml:"file_mem_backend"` - GuestHookPath string `toml:"guest_hook_path"` - GuestMemoryDumpPath string `toml:"guest_memory_dump_path"` - SeccompSandbox string `toml:"seccompsandbox"` - BlockDeviceAIO string `toml:"block_device_aio"` - HypervisorPathList []string `toml:"valid_hypervisor_paths"` - JailerPathList []string `toml:"valid_jailer_paths"` - CtlPathList []string `toml:"valid_ctlpaths"` - VirtioFSDaemonList []string `toml:"valid_virtio_fs_daemon_paths"` - VirtioFSExtraArgs []string `toml:"virtio_fs_extra_args"` - PFlashList []string `toml:"pflashes"` - VhostUserStorePathList []string `toml:"valid_vhost_user_store_paths"` - FileBackedMemRootList []string `toml:"valid_file_mem_backends"` - EntropySourceList []string `toml:"valid_entropy_sources"` - EnableAnnotations []string `toml:"enable_annotations"` - RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"` - TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"` - MemOffset uint64 `toml:"memory_offset"` - DefaultMaxMemorySize uint64 `toml:"default_maxmemory"` - DiskRateLimiterBwMaxRate int64 `toml:"disk_rate_limiter_bw_max_rate"` - DiskRateLimiterBwOneTimeBurst int64 `toml:"disk_rate_limiter_bw_one_time_burst"` - DiskRateLimiterOpsMaxRate int64 `toml:"disk_rate_limiter_ops_max_rate"` - DiskRateLimiterOpsOneTimeBurst int64 `toml:"disk_rate_limiter_ops_one_time_burst"` - NetRateLimiterBwMaxRate int64 `toml:"net_rate_limiter_bw_max_rate"` - NetRateLimiterBwOneTimeBurst int64 `toml:"net_rate_limiter_bw_one_time_burst"` - NetRateLimiterOpsMaxRate int64 `toml:"net_rate_limiter_ops_max_rate"` - NetRateLimiterOpsOneTimeBurst int64 `toml:"net_rate_limiter_ops_one_time_burst"` - VirtioFSCacheSize uint32 `toml:"virtio_fs_cache_size"` - VirtioFSQueueSize uint32 `toml:"virtio_fs_queue_size"` - DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` - MemorySize uint32 `toml:"default_memory"` - MemSlots uint32 `toml:"memory_slots"` - DefaultBridges uint32 `toml:"default_bridges"` - Msize9p uint32 `toml:"msize_9p"` - PCIeSwitchPort uint32 `toml:"pcie_switch_port"` - PCIeRootPort uint32 `toml:"pcie_root_port"` - NumVCPUs int32 `toml:"default_vcpus"` - BlockDeviceCacheSet bool `toml:"block_device_cache_set"` - BlockDeviceCacheDirect bool `toml:"block_device_cache_direct"` - BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"` - EnableVhostUserStore bool `toml:"enable_vhost_user_store"` - VhostUserDeviceReconnect uint32 `toml:"vhost_user_reconnect_timeout_sec"` - DisableBlockDeviceUse bool `toml:"disable_block_device_use"` - MemPrealloc bool `toml:"enable_mem_prealloc"` - HugePages bool `toml:"enable_hugepages"` - VirtioMem bool `toml:"enable_virtio_mem"` - IOMMU bool `toml:"enable_iommu"` - IOMMUPlatform bool `toml:"enable_iommu_platform"` - Debug bool `toml:"enable_debug"` - DisableNestingChecks bool `toml:"disable_nesting_checks"` - EnableIOThreads bool `toml:"enable_iothreads"` - DisableImageNvdimm bool `toml:"disable_image_nvdimm"` - HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"` - HotPlugVFIO hv.PCIePort `toml:"hot_plug_vfio"` - ColdPlugVFIO hv.PCIePort `toml:"cold_plug_vfio"` - DisableVhostNet bool `toml:"disable_vhost_net"` - GuestMemoryDumpPaging bool `toml:"guest_memory_dump_paging"` - ConfidentialGuest bool `toml:"confidential_guest"` - SevSnpGuest bool `toml:"sev_snp_guest"` - GuestSwap bool `toml:"enable_guest_swap"` - Rootless bool `toml:"rootless"` - DisableSeccomp bool `toml:"disable_seccomp"` - DisableSeLinux bool `toml:"disable_selinux"` - DisableGuestSeLinux bool `toml:"disable_guest_selinux"` - LegacySerial bool `toml:"use_legacy_serial"` + Path string `toml:"path"` + JailerPath string `toml:"jailer_path"` + Kernel string `toml:"kernel"` + CtlPath string `toml:"ctlpath"` + Initrd string `toml:"initrd"` + Image string `toml:"image"` + RootfsType string `toml:"rootfs_type"` + Firmware string `toml:"firmware"` + FirmwareVolume string `toml:"firmware_volume"` + MachineAccelerators string `toml:"machine_accelerators"` + CPUFeatures string `toml:"cpu_features"` + KernelParams string `toml:"kernel_params"` + MachineType string `toml:"machine_type"` + BlockDeviceDriver string `toml:"block_device_driver"` + EntropySource string `toml:"entropy_source"` + SharedFS string `toml:"shared_fs"` + VirtioFSDaemon string `toml:"virtio_fs_daemon"` + VirtioFSCache string `toml:"virtio_fs_cache"` + VhostUserStorePath string `toml:"vhost_user_store_path"` + FileBackedMemRootDir string `toml:"file_mem_backend"` + GuestHookPath string `toml:"guest_hook_path"` + GuestMemoryDumpPath string `toml:"guest_memory_dump_path"` + SeccompSandbox string `toml:"seccompsandbox"` + BlockDeviceAIO string `toml:"block_device_aio"` + HypervisorPathList []string `toml:"valid_hypervisor_paths"` + JailerPathList []string `toml:"valid_jailer_paths"` + CtlPathList []string `toml:"valid_ctlpaths"` + VirtioFSDaemonList []string `toml:"valid_virtio_fs_daemon_paths"` + VirtioFSExtraArgs []string `toml:"virtio_fs_extra_args"` + PFlashList []string `toml:"pflashes"` + VhostUserStorePathList []string `toml:"valid_vhost_user_store_paths"` + FileBackedMemRootList []string `toml:"valid_file_mem_backends"` + EntropySourceList []string `toml:"valid_entropy_sources"` + EnableAnnotations []string `toml:"enable_annotations"` + RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"` + TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"` + MemOffset uint64 `toml:"memory_offset"` + DefaultMaxMemorySize uint64 `toml:"default_maxmemory"` + DiskRateLimiterBwMaxRate int64 `toml:"disk_rate_limiter_bw_max_rate"` + DiskRateLimiterBwOneTimeBurst int64 `toml:"disk_rate_limiter_bw_one_time_burst"` + DiskRateLimiterOpsMaxRate int64 `toml:"disk_rate_limiter_ops_max_rate"` + DiskRateLimiterOpsOneTimeBurst int64 `toml:"disk_rate_limiter_ops_one_time_burst"` + NetRateLimiterBwMaxRate int64 `toml:"net_rate_limiter_bw_max_rate"` + NetRateLimiterBwOneTimeBurst int64 `toml:"net_rate_limiter_bw_one_time_burst"` + NetRateLimiterOpsMaxRate int64 `toml:"net_rate_limiter_ops_max_rate"` + NetRateLimiterOpsOneTimeBurst int64 `toml:"net_rate_limiter_ops_one_time_burst"` + VirtioFSCacheSize uint32 `toml:"virtio_fs_cache_size"` + VirtioFSQueueSize uint32 `toml:"virtio_fs_queue_size"` + DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` + MemorySize uint32 `toml:"default_memory"` + MemSlots uint32 `toml:"memory_slots"` + DefaultBridges uint32 `toml:"default_bridges"` + Msize9p uint32 `toml:"msize_9p"` + PCIeSwitchPort uint32 `toml:"pcie_switch_port"` + PCIeRootPort uint32 `toml:"pcie_root_port"` + NumVCPUs int32 `toml:"default_vcpus"` + BlockDeviceCacheSet bool `toml:"block_device_cache_set"` + BlockDeviceCacheDirect bool `toml:"block_device_cache_direct"` + BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"` + EnableVhostUserStore bool `toml:"enable_vhost_user_store"` + VhostUserDeviceReconnect uint32 `toml:"vhost_user_reconnect_timeout_sec"` + DisableBlockDeviceUse bool `toml:"disable_block_device_use"` + MemPrealloc bool `toml:"enable_mem_prealloc"` + HugePages bool `toml:"enable_hugepages"` + VirtioMem bool `toml:"enable_virtio_mem"` + IOMMU bool `toml:"enable_iommu"` + IOMMUPlatform bool `toml:"enable_iommu_platform"` + Debug bool `toml:"enable_debug"` + DisableNestingChecks bool `toml:"disable_nesting_checks"` + EnableIOThreads bool `toml:"enable_iothreads"` + DisableImageNvdimm bool `toml:"disable_image_nvdimm"` + HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"` + HotPlugVFIO config.PCIePort `toml:"hot_plug_vfio"` + ColdPlugVFIO config.PCIePort `toml:"cold_plug_vfio"` + DisableVhostNet bool `toml:"disable_vhost_net"` + GuestMemoryDumpPaging bool `toml:"guest_memory_dump_paging"` + ConfidentialGuest bool `toml:"confidential_guest"` + SevSnpGuest bool `toml:"sev_snp_guest"` + GuestSwap bool `toml:"enable_guest_swap"` + Rootless bool `toml:"rootless"` + DisableSeccomp bool `toml:"disable_seccomp"` + DisableSeLinux bool `toml:"disable_selinux"` + DisableGuestSeLinux bool `toml:"disable_guest_selinux"` + LegacySerial bool `toml:"use_legacy_serial"` } type runtime struct { @@ -289,13 +288,13 @@ func (h hypervisor) firmware() (string, error) { return ResolvePath(p) } -func (h hypervisor) coldPlugVFIO() hv.PCIePort { +func (h hypervisor) coldPlugVFIO() config.PCIePort { if h.ColdPlugVFIO == "" { return defaultColdPlugVFIO } return h.ColdPlugVFIO } -func (h hypervisor) hotPlugVFIO() hv.PCIePort { +func (h hypervisor) hotPlugVFIO() config.PCIePort { if h.HotPlugVFIO == "" { return defaultHotPlugVFIO } @@ -1689,29 +1688,29 @@ func checkConfig(config oci.RuntimeConfig) error { // checkPCIeConfig ensures the PCIe configuration is valid. // Only allow one of the following settings for cold-plug: // no-port, root-port, switch-port -func checkPCIeConfig(coldPlug hv.PCIePort, hotPlug hv.PCIePort, machineType string) error { +func checkPCIeConfig(coldPlug config.PCIePort, hotPlug config.PCIePort, machineType string) error { // Currently only QEMU q35 supports advanced PCIe topologies // firecracker, dragonball do not have right now any PCIe support if machineType != "q35" { return nil } - if coldPlug != hv.NoPort && hotPlug != hv.NoPort { + if coldPlug != config.NoPort && hotPlug != config.NoPort { return fmt.Errorf("invalid hot-plug=%s and cold-plug=%s settings, only one of them can be set", coldPlug, hotPlug) } - var port hv.PCIePort - if coldPlug != hv.NoPort { + var port config.PCIePort + if coldPlug != config.NoPort { port = coldPlug } - if hotPlug != hv.NoPort { + if hotPlug != config.NoPort { port = hotPlug } - if port == hv.NoPort || port == hv.BridgePort || port == hv.RootPort || port == hv.SwitchPort { + if port == config.NoPort || port == config.BridgePort || port == config.RootPort || port == config.SwitchPort { return nil } return fmt.Errorf("invalid vfio_port=%s setting, allowed values %s, %s, %s, %s", - coldPlug, hv.NoPort, hv.BridgePort, hv.RootPort, hv.SwitchPort) + coldPlug, config.NoPort, config.BridgePort, config.RootPort, config.SwitchPort) } // checkNetNsConfig performs sanity checks on disable_new_netns config. diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index f30a7a27d8..2f062bd08a 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -18,8 +18,8 @@ import ( "syscall" "testing" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" @@ -63,16 +63,16 @@ func createConfig(configPath string, fileData string) error { // createAllRuntimeConfigFiles creates all files necessary to call // loadConfiguration(). -func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConfig, err error) { +func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntimeConfig, err error) { if dir == "" { - return config, fmt.Errorf("BUG: need directory") + return testConfig, fmt.Errorf("BUG: need directory") } if hypervisor == "" { - return config, fmt.Errorf("BUG: need hypervisor") + return testConfig, fmt.Errorf("BUG: need hypervisor") } - var hotPlugVFIO hv.PCIePort - var coldPlugVFIO hv.PCIePort + var hotPlugVFIO config.PCIePort + var coldPlugVFIO config.PCIePort hypervisorPath := path.Join(dir, "hypervisor") kernelPath := path.Join(dir, "kernel") kernelParams := "foo=bar xyz" @@ -88,8 +88,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf hotplugVFIOOnRootBus := true pcieRootPort := uint32(2) pcieSwitchPort := uint32(3) - hotPlugVFIO = hv.BridgePort - coldPlugVFIO = hv.RootPort + hotPlugVFIO = config.BridgePort + coldPlugVFIO = config.RootPort disableNewNetNs := false sharedFS := "virtio-9p" virtioFSdaemon := path.Join(dir, "virtiofsd") @@ -139,7 +139,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf configPath := path.Join(dir, "runtime.toml") err = createConfig(configPath, runtimeConfigFileData) if err != nil { - return config, err + return testConfig, err } configPathLink := path.Join(filepath.Dir(configPath), "link-to-configuration.toml") @@ -147,7 +147,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf // create a link to the config file err = syscall.Symlink(configPath, configPathLink) if err != nil { - return config, err + return testConfig, err } files := []string{hypervisorPath, kernelPath, imagePath} @@ -156,7 +156,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf // create the resource (which must be >0 bytes) err := WriteFile(file, "foo", testFileMode) if err != nil { - return config, err + return testConfig, err } } @@ -223,10 +223,10 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf err = SetKernelParams(&runtimeConfig) if err != nil { - return config, err + return testConfig, err } - config = testRuntimeConfig{ + rtimeConfig := testRuntimeConfig{ RuntimeConfig: runtimeConfig, RuntimeConfigFile: configPath, ConfigPath: configPath, @@ -235,7 +235,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf LogPath: logPath, } - return config, nil + return rtimeConfig, nil } // testLoadConfiguration accepts an optional function that can be used @@ -610,7 +610,7 @@ func TestMinimalRuntimeConfig(t *testing.T) { func TestNewQemuHypervisorConfig(t *testing.T) { dir := t.TempDir() - var coldPlugVFIO hv.PCIePort + var coldPlugVFIO config.PCIePort hypervisorPath := path.Join(dir, "hypervisor") kernelPath := path.Join(dir, "kernel") imagePath := path.Join(dir, "image") @@ -619,7 +619,7 @@ func TestNewQemuHypervisorConfig(t *testing.T) { enableIOThreads := true hotplugVFIOOnRootBus := true pcieRootPort := uint32(2) - coldPlugVFIO = hv.RootPort + coldPlugVFIO = config.RootPort orgVHostVSockDevicePath := utils.VHostVSockDevicePath blockDeviceAIO := "io_uring" defer func() { diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 43974adf68..62a8d866f5 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -511,11 +511,11 @@ type HypervisorConfig struct { // HotplugVFIO is used to indicate if devices need to be hotplugged on the // root port or a switch - HotPlugVFIO hv.PCIePort + HotPlugVFIO config.PCIePort // ColdPlugVFIO is used to indicate if devices need to be coldplugged on the // root port, switch or no port - ColdPlugVFIO hv.PCIePort + ColdPlugVFIO config.PCIePort // PCIeRootPort is used to indicate the number of PCIe Root Port devices // The PCIe Root Port device is used to hot-plug the PCIe device diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index 3653674ca9..c7fa059dcb 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -1048,16 +1048,16 @@ func TestKataAgentKernelParams(t *testing.T) { {true, true, 0, []Param{debugParam, traceParam}}, // pipesize - {false, false, 0, []Param{containerPipeSizeParam}}, + {false, false, 2097152, []Param{containerPipeSizeParam}}, // Debug + pipesize - {true, false, 0, []Param{debugParam, containerPipeSizeParam}}, + {true, false, 2097152, []Param{debugParam, containerPipeSizeParam}}, // Tracing + pipesize - {false, true, 0, []Param{traceParam, containerPipeSizeParam}}, + {false, true, 2097152, []Param{traceParam, containerPipeSizeParam}}, // Debug + Tracing + pipesize - {true, true, 0, []Param{debugParam, traceParam, containerPipeSizeParam}}, + {true, true, 2097152, []Param{debugParam, traceParam, containerPipeSizeParam}}, } for i, d := range data { diff --git a/src/runtime/virtcontainers/nydusd_test.go b/src/runtime/virtcontainers/nydusd_test.go index 712113015c..a8ec6dc9b1 100644 --- a/src/runtime/virtcontainers/nydusd_test.go +++ b/src/runtime/virtcontainers/nydusd_test.go @@ -57,13 +57,13 @@ func TestNydusdStart(t *testing.T) { // nolint: govet tests := []struct { - fields fields name string + fields fields wantErr bool }{ - {name: "empty config", fields: fields{}, wantErr: true}, - {name: "directory source path not exist", fields: SourcePathNoExist, wantErr: true}, - {name: "valid config", fields: validConfig, wantErr: false}, + {"empty config", fields{}, true}, + {"directory source path not exist", SourcePathNoExist, true}, + {"valid config", validConfig, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go index 8cd7520937..aeb347127c 100644 --- a/src/runtime/virtcontainers/persist/api/config.go +++ b/src/runtime/virtcontainers/persist/api/config.go @@ -7,7 +7,7 @@ package persistapi import ( - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/opencontainers/runc/libcontainer/configs" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -205,11 +205,11 @@ type HypervisorConfig struct { // HotPlugVFIO is used to indicate if devices need to be hotplugged on the // root, switch, bridge or no-port - HotPlugVFIO hv.PCIePort + HotPlugVFIO config.PCIePort // ColdPlugVFIO is used to indicate if devices need to be coldplugged on the // root, bridge, switch or no-port - ColdPlugVFIO hv.PCIePort + ColdPlugVFIO config.PCIePort // BootToBeTemplate used to indicate if the VM is created to be a template VM BootToBeTemplate bool diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 514cc9ee3c..acd5fcc7e7 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -76,7 +76,7 @@ type qmpChannel struct { // QemuState keeps Qemu's state type QemuState struct { UUID string - HotPlugVFIO hv.PCIePort + HotPlugVFIO config.PCIePort Bridges []types.Bridge HotpluggedVCPUs []hv.CPUDevice HotpluggedMemory int @@ -84,8 +84,8 @@ type QemuState struct { PCIeRootPort int PCIeSwitchPort int HotplugVFIOOnRootBus bool - HotplugVFIO hv.PCIePort - ColdPlugVFIO hv.PCIePort + HotplugVFIO config.PCIePort + ColdPlugVFIO config.PCIePort } // qemu is an Hypervisor interface implementation for the Linux qemu hypervisor. @@ -748,7 +748,7 @@ func (q *qemu) checkBpfEnabled() { // only 16 ports possible. func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig *HypervisorConfig) error { // If no-port set just return no need to add PCIe Root Port or PCIe Switches - if hypervisorConfig.HotPlugVFIO == hv.NoPort && hypervisorConfig.ColdPlugVFIO == hv.NoPort { + if hypervisorConfig.HotPlugVFIO == config.NoPort && hypervisorConfig.ColdPlugVFIO == config.NoPort { return nil } // Add PCIe Root Port or PCIe Switches to the hypervisor @@ -787,11 +787,14 @@ func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig if drivers.IsPCIeDevice(vfioDevice.BDF) { numOfPluggablePorts = numOfPluggablePorts + 1 } - // Reset the Rank, the vfio module is going to assign the correct one - vfioDevice.Rank = -1 + // Reset the Rank and AllPCIeDevsthe vfio module is going + // to assign the correct one in the case of hot-plug + if hypervisorConfig.HotPlugVFIO != config.NoPort { + vfioDevice.Rank = -1 + drivers.AllPCIeDevs = map[string]bool{} + } } } - drivers.AllPCIeDevs = make(map[string]bool) // If number of PCIe root ports > 16 then bail out otherwise we may // use up all slots or IO memory on the root bus and vfio-XXX-pci devices @@ -815,10 +818,10 @@ func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig numOfPluggablePorts = hypervisorConfig.PCIeSwitchPort } - if q.state.HotPlugVFIO == hv.RootPort || q.state.ColdPlugVFIO == hv.RootPort || q.state.HotplugVFIOOnRootBus { + if q.state.HotPlugVFIO == config.RootPort || q.state.ColdPlugVFIO == config.RootPort || q.state.HotplugVFIOOnRootBus { qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, numOfPluggablePorts, memSize32bit, memSize64bit) } - if q.state.HotPlugVFIO == hv.SwitchPort || q.state.ColdPlugVFIO == hv.SwitchPort { + if q.state.HotPlugVFIO == config.SwitchPort || q.state.ColdPlugVFIO == config.SwitchPort { qemuConfig.Devices = q.arch.appendPCIeSwitchPortDevice(qemuConfig.Devices, numOfPluggablePorts, memSize32bit, memSize64bit) } return nil @@ -1629,7 +1632,7 @@ func (q *qemu) hotplugAddVhostUserBlkDevice(ctx context.Context, vAttr *config.V //Since the dev is the first and only one on this bus(root port), it should be 0. addr := "00" - bridgeID := fmt.Sprintf("%s%d", pcieRootPortPrefix, len(drivers.AllPCIeDevs)) + bridgeID := fmt.Sprintf("%s%d", config.PCIeRootPortPrefix, len(drivers.AllPCIeDevs)) drivers.AllPCIeDevs[devID] = true bridgeQomPath := fmt.Sprintf("%s%s", qomPathPrefix, bridgeID) @@ -1824,7 +1827,6 @@ func (q *qemu) hotplugVFIODeviceRootPort(ctx context.Context, device *config.VFI q.state.HotPlugVFIO, q.state.PCIeRootPort, q.state.PCIeSwitchPort) } - device.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, device.Rank) return q.executeVFIODeviceAdd(device) } @@ -1837,7 +1839,6 @@ func (q *qemu) hotplugVFIODeviceSwitchPort(ctx context.Context, device *config.V q.state.HotPlugVFIO, q.state.PCIeRootPort, q.state.PCIeSwitchPort) } - device.Bus = fmt.Sprintf("%s%d", pcieSwitchDownstreamPortPrefix, device.Rank) return q.executeVFIODeviceAdd(device) } @@ -1904,9 +1905,9 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op // In case HotplugVFIOOnRootBus is true, devices are hotplugged on the root bus // for pc machine type instead of bridge. This is useful for devices that require // a large PCI BAR which is a currently a limitation with PCI bridges. - if q.state.HotPlugVFIO == hv.RootPort || q.state.HotplugVFIOOnRootBus { + if q.state.HotPlugVFIO == config.RootPort || q.state.HotplugVFIOOnRootBus { err = q.hotplugVFIODeviceRootPort(ctx, device) - } else if q.state.HotPlugVFIO == hv.SwitchPort { + } else if q.state.HotPlugVFIO == config.SwitchPort { err = q.hotplugVFIODeviceSwitchPort(ctx, device) } else { err = q.hotplugVFIODeviceBridgePort(ctx, device) @@ -2632,7 +2633,7 @@ func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machin for i := uint32(0); i < number; i++ { devices = append(devices, govmmQemu.PCIeRootPortDevice{ - ID: fmt.Sprintf("%s%d", pcieRootPortPrefix, i), + ID: fmt.Sprintf("%s%d", config.PCIeRootPortPrefix, i), Bus: bus, Chassis: chassis, Slot: strconv.FormatUint(uint64(i), 10), @@ -2679,7 +2680,7 @@ func genericAppendPCIeSwitchPort(devices []govmmQemu.Device, number uint32, mach // Using an own ID for the root port, so we do not clash with already // existing root ports adding "s" for switch prefix pcieRootPort := govmmQemu.PCIeRootPortDevice{ - ID: fmt.Sprintf("%s%s%d", pcieSwitchPrefix, pcieRootPortPrefix, 0), + ID: fmt.Sprintf("%s%s%d", config.PCIeSwitchPortPrefix, config.PCIeRootPortPrefix, 0), Bus: defaultBridgeBus, Chassis: "0", Slot: strconv.FormatUint(uint64(0), 10), @@ -2692,7 +2693,7 @@ func genericAppendPCIeSwitchPort(devices []govmmQemu.Device, number uint32, mach devices = append(devices, pcieRootPort) pcieSwitchUpstreamPort := govmmQemu.PCIeSwitchUpstreamPortDevice{ - ID: fmt.Sprintf("%s%d", pcieSwitchUpstreamPortPrefix, 0), + ID: fmt.Sprintf("%s%d", config.PCIeSwitchUpstreamPortPrefix, 0), Bus: pcieRootPort.ID, } devices = append(devices, pcieSwitchUpstreamPort) @@ -2706,7 +2707,7 @@ func genericAppendPCIeSwitchPort(devices []govmmQemu.Device, number uint32, mach for i := uint32(0); i < number; i++ { pcieSwitchDownstreamPort := govmmQemu.PCIeSwitchDownstreamPortDevice{ - ID: fmt.Sprintf("%s%d", pcieSwitchDownstreamPortPrefix, i), + ID: fmt.Sprintf("%s%d", config.PCIeSwitchhDownstreamPortPrefix, i), Bus: pcieSwitchUpstreamPort.ID, Chassis: fmt.Sprintf("%d", nextChassis), Slot: strconv.FormatUint(uint64(i), 10), diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index 9a288d85a7..36a23371fc 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -180,18 +180,14 @@ type qemuArchBase struct { } const ( - defaultCores uint32 = 1 - defaultThreads uint32 = 1 - defaultCPUModel = "host" - defaultBridgeBus = "pcie.0" - defaultPCBridgeBus = "pci.0" - maxDevIDSize = 31 - maxPCIeRootPort = 16 // Limitation from QEMU - maxPCIeSwitchPort = 16 // Limitation from QEMU - pcieRootPortPrefix = "rp" - pcieSwitchPrefix = "sw" - pcieSwitchUpstreamPortPrefix = "swup" - pcieSwitchDownstreamPortPrefix = "swdp" + defaultCores uint32 = 1 + defaultThreads uint32 = 1 + defaultCPUModel = "host" + defaultBridgeBus = "pcie.0" + defaultPCBridgeBus = "pci.0" + maxDevIDSize = 31 + maxPCIeRootPort = 16 // Limitation from QEMU + maxPCIeSwitchPort = 16 // Limitation from QEMU ) // This is the PCI start address assigned to the first bridge that diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index a5e9ca4a5f..9fe88b28ce 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -32,7 +32,6 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers" deviceManager "github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager" - hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol" exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental" @@ -616,10 +615,10 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor // If we have a confidential guest we need to cold-plug the PCIe VFIO devices // until we have TDISP/IDE PCIe support. - coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO != hv.NoPort) + coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO != config.NoPort) // Aggregate all the containner devices for hot-plug and use them to dedcue // the correct amount of ports to reserve for the hypervisor. - hotPlugVFIO := (sandboxConfig.HypervisorConfig.HotPlugVFIO != hv.NoPort) + hotPlugVFIO := (sandboxConfig.HypervisorConfig.HotPlugVFIO != config.NoPort) var vfioHotPlugDevices []config.DeviceInfo var vfioColdPlugDevices []config.DeviceInfo @@ -629,9 +628,11 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor isVFIO := deviceManager.IsVFIO(device.ContainerPath) if hotPlugVFIO && isVFIO { vfioHotPlugDevices = append(vfioHotPlugDevices, device) + sandboxConfig.Containers[cnt].DeviceInfos[dev].Port = sandboxConfig.HypervisorConfig.HotPlugVFIO } if coldPlugVFIO && isVFIO { device.ColdPlug = true + device.Port = sandboxConfig.HypervisorConfig.ColdPlugVFIO vfioColdPlugDevices = append(vfioColdPlugDevices, device) // We need to remove the devices marked for cold-plug // otherwise at the container level the kata-agent @@ -639,6 +640,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor infos := sandboxConfig.Containers[cnt].DeviceInfos infos = append(infos[:dev], infos[dev+1:]...) sandboxConfig.Containers[cnt].DeviceInfos = infos + } } } @@ -2015,7 +2017,9 @@ func (s *Sandbox) AppendDevice(ctx context.Context, device api.Device) error { return s.hypervisor.AddDevice(ctx, device.GetDeviceInfo().(*config.VhostUserDeviceAttrs), VhostuserDev) case config.DeviceVFIO: vfioDevs := device.GetDeviceInfo().([]*config.VFIODev) + s.Logger().Info("### vfioDevs: ", vfioDevs) for _, d := range vfioDevs { + s.Logger().Info("### vfioDev: ", d) return s.hypervisor.AddDevice(ctx, *d, VfioDev) } default: