gpu: Moved the PCIe configs to drivers

The hypervisor_state file was the wrong location for the PCIe Port
settings, moved everything under device umbrella, where it can be
consumed more easily and we do not get into circular deps.

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>

src/runtime/cmd/kata-runtime/kata-env.go

@@ -17,7 +17,7 @@ import (
 	"github.com/prometheus/procfs"
 	"github.com/urfave/cli"
 
-	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
@@ -115,8 +115,8 @@ type HypervisorInfo struct {
 	MemorySlots          uint32
 	PCIeRootPort         uint32
 	PCIeSwitchPort       uint32
-	HotPlugVFIO          hv.PCIePort
+	HotPlugVFIO          config.PCIePort
-	ColdPlugVFIO         hv.PCIePort
+	ColdPlugVFIO         config.PCIePort
 	HotplugVFIOOnRootBus bool
 	Debug                bool
 }

src/runtime/cmd/kata-runtime/kata-env_test.go

@@ -19,12 +19,12 @@ import (
 	"testing"
 
 	"github.com/BurntSushi/toml"
-	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	vcUtils "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/urfave/cli"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
@@ -74,9 +74,9 @@ func createConfig(configPath string, fileData string) error {
 	return nil
 }
 
-func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeConfig, err error) {
+func makeRuntimeConfig(prefixDir string) (configFile string, ociConfig oci.RuntimeConfig, err error) {
-	var hotPlugVFIO hv.PCIePort
+	var hotPlugVFIO config.PCIePort
-	var coldPlugVFIO hv.PCIePort
+	var coldPlugVFIO config.PCIePort
 	const logPath = "/log/path"
 	hypervisorPath := filepath.Join(prefixDir, "hypervisor")
 	kernelPath := filepath.Join(prefixDir, "kernel")
@@ -90,8 +90,8 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 	hotplugVFIOOnRootBus := true
 	pcieRootPort := uint32(2)
 	pcieSwitchPort := uint32(2)
-	hotPlugVFIO = hv.BridgePort
+	hotPlugVFIO = config.BridgePort
-	coldPlugVFIO = hv.NoPort
+	coldPlugVFIO = config.NoPort
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
 	virtioFSdaemon := filepath.Join(prefixDir, "virtiofsd")
@@ -161,12 +161,12 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 		return "", oci.RuntimeConfig{}, err
 	}
 
-	_, config, err = katautils.LoadConfiguration(configFile, true)
+	_, ociConfig, err = katautils.LoadConfiguration(configFile, true)
 	if err != nil {
 		return "", oci.RuntimeConfig{}, err
 	}
 
-	return configFile, config, nil
+	return configFile, ociConfig, nil
 }
 
 func getExpectedAgentDetails(config oci.RuntimeConfig) (AgentInfo, error) {

src/runtime/pkg/containerd-shim-v2/create_test.go

@@ -20,7 +20,7 @@ import (
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"
 
-	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
@@ -308,9 +308,9 @@ func TestCreateContainerConfigFail(t *testing.T) {
 	assert.Error(err)
 }
 
-func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err error) {
+func createAllRuntimeConfigFiles(dir, hypervisor string) (runtimeConfig string, err error) {
-	var hotPlugVFIO hv.PCIePort
+	var hotPlugVFIO config.PCIePort
-	var coldPlugVFIO hv.PCIePort
+	var coldPlugVFIO config.PCIePort
 	if dir == "" {
 		return "", fmt.Errorf("BUG: need directory")
 	}
@@ -336,8 +336,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
 	virtioFSdaemon := path.Join(dir, "virtiofsd")
-	hotPlugVFIO = hv.BridgePort
+	hotPlugVFIO = config.BridgePort
-	coldPlugVFIO = hv.RootPort
+	coldPlugVFIO = config.RootPort
 
 	configFileOptions := ktu.RuntimeConfigOptions{
 		Hypervisor:           "qemu",

src/runtime/pkg/device/config/config.go

@@ -122,6 +122,68 @@ var SysBusPciDevicesPath = "/sys/bus/pci/devices"
 
 var getSysDevPath = getSysDevPathImpl
 
+// PCIePortBusPrefix gives us the correct bus nameing dependeing on the port
+// used to hot(cold)-plug the device
+type PCIePortBusPrefix string
+
+const (
+	PCIeRootPortPrefix              PCIePortBusPrefix = "rp"
+	PCIeSwitchPortPrefix            PCIePortBusPrefix = "sw"
+	PCIeSwitchUpstreamPortPrefix    PCIePortBusPrefix = "swup"
+	PCIeSwitchhDownstreamPortPrefix PCIePortBusPrefix = "swdp"
+	PCIBridgePortPrefix             PCIePortBusPrefix = "bp"
+)
+
+func (p PCIePortBusPrefix) String() string {
+	switch p {
+	case PCIeRootPortPrefix:
+		fallthrough
+	case PCIeSwitchPortPrefix:
+		fallthrough
+	case PCIeSwitchUpstreamPortPrefix:
+		fallthrough
+	case PCIeSwitchhDownstreamPortPrefix:
+		fallthrough
+	case PCIBridgePortPrefix:
+		return string(p)
+	}
+	return fmt.Sprintf("<unknown PCIePortBusPrefix: %s>", string(p))
+}
+
+// PCIePort distinguish only between root and switch port
+type PCIePort string
+
+const (
+	// RootPort attach VFIO devices to a root-port
+	RootPort PCIePort = "root-port"
+	// SwitchPort attach VFIO devices to a switch-port
+	SwitchPort = "switch-port"
+	// BridgePort is the default
+	BridgePort = "bridge-port"
+	// NoPort is for disabling VFIO hotplug/coldplug
+	NoPort = "no-port"
+)
+
+func (p PCIePort) String() string {
+	switch p {
+	case RootPort:
+		fallthrough
+	case SwitchPort:
+		fallthrough
+	case BridgePort:
+		fallthrough
+	case NoPort:
+		return string(p)
+	}
+	return fmt.Sprintf("<unknown PCIePort: %s>", string(p))
+}
+
+var PCIePortPrefixMapping = map[PCIePort]PCIePortBusPrefix{
+	RootPort:   PCIeRootPortPrefix,
+	SwitchPort: PCIeSwitchhDownstreamPortPrefix,
+	BridgePort: PCIBridgePortPrefix,
+}
+
 // DeviceInfo is an embedded type that contains device data common to all types of devices.
 type DeviceInfo struct {
 	// DriverOptions is specific options for each device driver
@@ -167,6 +229,9 @@ type DeviceInfo struct {
 	// ColdPlug specifies whether the device must be cold plugged (true)
 	// or hot plugged (false).
 	ColdPlug bool
+
+	// Specifies the PCIe port type to which the device is attached
+	Port PCIePort
 }
 
 // BlockDrive represents a block storage drive which may be used in case the storage

src/runtime/pkg/device/drivers/utils.go

@@ -208,7 +208,9 @@ func GetAllVFIODevicesFromIOMMUGroup(device config.DeviceInfo) ([]*config.VFIODe
 			if isPCIe {
 				vfioPCI.Rank = len(AllPCIeDevs)
 				AllPCIeDevs[deviceBDF] = true
+				vfioPCI.Bus = fmt.Sprintf("%s%d", config.PCIePortPrefixMapping[device.Port], vfioPCI.Rank)
 			}
+
 			vfio = vfioPCI
 		case config.VFIOAPDeviceMediatedType:
 			devices, err := GetAPVFIODevices(deviceSysfsDev)

src/runtime/pkg/device/drivers/vfio.go

@@ -28,7 +28,6 @@ const (
 	vfioRemoveIDPath    = "/sys/bus/pci/drivers/vfio-pci/remove_id"
 	iommuGroupPath      = "/sys/bus/pci/devices/%s/iommu_group"
 	vfioDevPath         = "/dev/vfio/%s"
-	pcieRootPortPrefix  = "rp"
 	vfioAPSysfsDir      = "/sys/devices/vfio_ap"
 )
 

src/runtime/pkg/hypervisors/hypervisor_state.go

@@ -5,7 +5,7 @@
 
 package hypervisors
 
-import "fmt"
+import "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 
 // Bridge is a bridge where devices can be hot plugged
 type Bridge struct {
@@ -28,34 +28,6 @@ type CPUDevice struct {
 	ID string
 }
 
-// PCIePort distinguish only between root and switch port
-type PCIePort string
-
-const (
-	// RootPort attach VFIO devices to a root-port
-	RootPort PCIePort = "root-port"
-	// SwitchPort attach VFIO devices to a switch-port
-	SwitchPort = "switch-port"
-	// BridgePort is the default
-	BridgePort = "bridge-port"
-	// NoPort is for disabling VFIO hotplug/coldplug
-	NoPort = "no-port"
-)
-
-func (p PCIePort) String() string {
-	switch p {
-	case RootPort:
-		fallthrough
-	case SwitchPort:
-		fallthrough
-	case BridgePort:
-		fallthrough
-	case NoPort:
-		return string(p)
-	}
-	return fmt.Sprintf("<unknown PCIePort: %s>", string(p))
-}
-
 type HypervisorState struct {
 	BlockIndexMap map[int]struct{}
 	// Type of hypervisor, E.g. qemu/firecracker/acrn.
@@ -76,7 +48,7 @@ type HypervisorState struct {
 	Pid                  int
 	PCIeRootPort         int
 	PCIeSwitchPort       int
-	HotPlugVFIO          PCIePort
+	HotPlugVFIO          config.PCIePort
-	ColdPlugVFIO         PCIePort
+	ColdPlugVFIO         config.PCIePort
 	HotplugVFIOOnRootBus bool
 }

src/runtime/pkg/katatestutils/utils.go

@@ -14,7 +14,7 @@ import (
 	"strconv"
 	"testing"
 
-	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"
 )
@@ -226,8 +226,8 @@ type RuntimeConfigOptions struct {
 	PFlash               []string
 	PCIeRootPort         uint32
 	PCIeSwitchPort       uint32
-	HotPlugVFIO          hv.PCIePort
+	HotPlugVFIO          config.PCIePort
-	ColdPlugVFIO         hv.PCIePort
+	ColdPlugVFIO         config.PCIePort
 	DefaultVCPUCount     uint32
 	DefaultMaxVCPUCount  uint32
 	DefaultMemSize       uint32

src/runtime/pkg/katautils/config-settings.go.in

@@ -10,7 +10,7 @@
 package katautils
 
 import (
-	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
+	config "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 )
 
 // name is the name of the runtime
@@ -109,7 +109,7 @@ const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock"
 // Default config file used by stateless systems.
 var defaultRuntimeConfiguration = "@CONFIG_PATH@"
 
-const defaultHotPlugVFIO = hv.NoPort
+const defaultHotPlugVFIO = config.NoPort
-const defaultColdPlugVFIO = hv.NoPort
+const defaultColdPlugVFIO = config.NoPort
 
 

src/runtime/pkg/katautils/config.go

@@ -20,7 +20,6 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
 	govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
-	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
@@ -78,90 +77,90 @@ type factory struct {
 }
 
 type hypervisor struct {
-	Path                           string      `toml:"path"`
+	Path                           string          `toml:"path"`
-	JailerPath                     string      `toml:"jailer_path"`
+	JailerPath                     string          `toml:"jailer_path"`
-	Kernel                         string      `toml:"kernel"`
+	Kernel                         string          `toml:"kernel"`
-	CtlPath                        string      `toml:"ctlpath"`
+	CtlPath                        string          `toml:"ctlpath"`
-	Initrd                         string      `toml:"initrd"`
+	Initrd                         string          `toml:"initrd"`
-	Image                          string      `toml:"image"`
+	Image                          string          `toml:"image"`
-	RootfsType                     string      `toml:"rootfs_type"`
+	RootfsType                     string          `toml:"rootfs_type"`
-	Firmware                       string      `toml:"firmware"`
+	Firmware                       string          `toml:"firmware"`
-	FirmwareVolume                 string      `toml:"firmware_volume"`
+	FirmwareVolume                 string          `toml:"firmware_volume"`
-	MachineAccelerators            string      `toml:"machine_accelerators"`
+	MachineAccelerators            string          `toml:"machine_accelerators"`
-	CPUFeatures                    string      `toml:"cpu_features"`
+	CPUFeatures                    string          `toml:"cpu_features"`
-	KernelParams                   string      `toml:"kernel_params"`
+	KernelParams                   string          `toml:"kernel_params"`
-	MachineType                    string      `toml:"machine_type"`
+	MachineType                    string          `toml:"machine_type"`
-	BlockDeviceDriver              string      `toml:"block_device_driver"`
+	BlockDeviceDriver              string          `toml:"block_device_driver"`
-	EntropySource                  string      `toml:"entropy_source"`
+	EntropySource                  string          `toml:"entropy_source"`
-	SharedFS                       string      `toml:"shared_fs"`
+	SharedFS                       string          `toml:"shared_fs"`
-	VirtioFSDaemon                 string      `toml:"virtio_fs_daemon"`
+	VirtioFSDaemon                 string          `toml:"virtio_fs_daemon"`
-	VirtioFSCache                  string      `toml:"virtio_fs_cache"`
+	VirtioFSCache                  string          `toml:"virtio_fs_cache"`
-	VhostUserStorePath             string      `toml:"vhost_user_store_path"`
+	VhostUserStorePath             string          `toml:"vhost_user_store_path"`
-	FileBackedMemRootDir           string      `toml:"file_mem_backend"`
+	FileBackedMemRootDir           string          `toml:"file_mem_backend"`
-	GuestHookPath                  string      `toml:"guest_hook_path"`
+	GuestHookPath                  string          `toml:"guest_hook_path"`
-	GuestMemoryDumpPath            string      `toml:"guest_memory_dump_path"`
+	GuestMemoryDumpPath            string          `toml:"guest_memory_dump_path"`
-	SeccompSandbox                 string      `toml:"seccompsandbox"`
+	SeccompSandbox                 string          `toml:"seccompsandbox"`
-	BlockDeviceAIO                 string      `toml:"block_device_aio"`
+	BlockDeviceAIO                 string          `toml:"block_device_aio"`
-	HypervisorPathList             []string    `toml:"valid_hypervisor_paths"`
+	HypervisorPathList             []string        `toml:"valid_hypervisor_paths"`
-	JailerPathList                 []string    `toml:"valid_jailer_paths"`
+	JailerPathList                 []string        `toml:"valid_jailer_paths"`
-	CtlPathList                    []string    `toml:"valid_ctlpaths"`
+	CtlPathList                    []string        `toml:"valid_ctlpaths"`
-	VirtioFSDaemonList             []string    `toml:"valid_virtio_fs_daemon_paths"`
+	VirtioFSDaemonList             []string        `toml:"valid_virtio_fs_daemon_paths"`
-	VirtioFSExtraArgs              []string    `toml:"virtio_fs_extra_args"`
+	VirtioFSExtraArgs              []string        `toml:"virtio_fs_extra_args"`
-	PFlashList                     []string    `toml:"pflashes"`
+	PFlashList                     []string        `toml:"pflashes"`
-	VhostUserStorePathList         []string    `toml:"valid_vhost_user_store_paths"`
+	VhostUserStorePathList         []string        `toml:"valid_vhost_user_store_paths"`
-	FileBackedMemRootList          []string    `toml:"valid_file_mem_backends"`
+	FileBackedMemRootList          []string        `toml:"valid_file_mem_backends"`
-	EntropySourceList              []string    `toml:"valid_entropy_sources"`
+	EntropySourceList              []string        `toml:"valid_entropy_sources"`
-	EnableAnnotations              []string    `toml:"enable_annotations"`
+	EnableAnnotations              []string        `toml:"enable_annotations"`
-	RxRateLimiterMaxRate           uint64      `toml:"rx_rate_limiter_max_rate"`
+	RxRateLimiterMaxRate           uint64          `toml:"rx_rate_limiter_max_rate"`
-	TxRateLimiterMaxRate           uint64      `toml:"tx_rate_limiter_max_rate"`
+	TxRateLimiterMaxRate           uint64          `toml:"tx_rate_limiter_max_rate"`
-	MemOffset                      uint64      `toml:"memory_offset"`
+	MemOffset                      uint64          `toml:"memory_offset"`
-	DefaultMaxMemorySize           uint64      `toml:"default_maxmemory"`
+	DefaultMaxMemorySize           uint64          `toml:"default_maxmemory"`
-	DiskRateLimiterBwMaxRate       int64       `toml:"disk_rate_limiter_bw_max_rate"`
+	DiskRateLimiterBwMaxRate       int64           `toml:"disk_rate_limiter_bw_max_rate"`
-	DiskRateLimiterBwOneTimeBurst  int64       `toml:"disk_rate_limiter_bw_one_time_burst"`
+	DiskRateLimiterBwOneTimeBurst  int64           `toml:"disk_rate_limiter_bw_one_time_burst"`
-	DiskRateLimiterOpsMaxRate      int64       `toml:"disk_rate_limiter_ops_max_rate"`
+	DiskRateLimiterOpsMaxRate      int64           `toml:"disk_rate_limiter_ops_max_rate"`
-	DiskRateLimiterOpsOneTimeBurst int64       `toml:"disk_rate_limiter_ops_one_time_burst"`
+	DiskRateLimiterOpsOneTimeBurst int64           `toml:"disk_rate_limiter_ops_one_time_burst"`
-	NetRateLimiterBwMaxRate        int64       `toml:"net_rate_limiter_bw_max_rate"`
+	NetRateLimiterBwMaxRate        int64           `toml:"net_rate_limiter_bw_max_rate"`
-	NetRateLimiterBwOneTimeBurst   int64       `toml:"net_rate_limiter_bw_one_time_burst"`
+	NetRateLimiterBwOneTimeBurst   int64           `toml:"net_rate_limiter_bw_one_time_burst"`
-	NetRateLimiterOpsMaxRate       int64       `toml:"net_rate_limiter_ops_max_rate"`
+	NetRateLimiterOpsMaxRate       int64           `toml:"net_rate_limiter_ops_max_rate"`
-	NetRateLimiterOpsOneTimeBurst  int64       `toml:"net_rate_limiter_ops_one_time_burst"`
+	NetRateLimiterOpsOneTimeBurst  int64           `toml:"net_rate_limiter_ops_one_time_burst"`
-	VirtioFSCacheSize              uint32      `toml:"virtio_fs_cache_size"`
+	VirtioFSCacheSize              uint32          `toml:"virtio_fs_cache_size"`
-	VirtioFSQueueSize              uint32      `toml:"virtio_fs_queue_size"`
+	VirtioFSQueueSize              uint32          `toml:"virtio_fs_queue_size"`
-	DefaultMaxVCPUs                uint32      `toml:"default_maxvcpus"`
+	DefaultMaxVCPUs                uint32          `toml:"default_maxvcpus"`
-	MemorySize                     uint32      `toml:"default_memory"`
+	MemorySize                     uint32          `toml:"default_memory"`
-	MemSlots                       uint32      `toml:"memory_slots"`
+	MemSlots                       uint32          `toml:"memory_slots"`
-	DefaultBridges                 uint32      `toml:"default_bridges"`
+	DefaultBridges                 uint32          `toml:"default_bridges"`
-	Msize9p                        uint32      `toml:"msize_9p"`
+	Msize9p                        uint32          `toml:"msize_9p"`
-	PCIeSwitchPort                 uint32      `toml:"pcie_switch_port"`
+	PCIeSwitchPort                 uint32          `toml:"pcie_switch_port"`
-	PCIeRootPort                   uint32      `toml:"pcie_root_port"`
+	PCIeRootPort                   uint32          `toml:"pcie_root_port"`
-	NumVCPUs                       int32       `toml:"default_vcpus"`
+	NumVCPUs                       int32           `toml:"default_vcpus"`
-	BlockDeviceCacheSet            bool        `toml:"block_device_cache_set"`
+	BlockDeviceCacheSet            bool            `toml:"block_device_cache_set"`
-	BlockDeviceCacheDirect         bool        `toml:"block_device_cache_direct"`
+	BlockDeviceCacheDirect         bool            `toml:"block_device_cache_direct"`
-	BlockDeviceCacheNoflush        bool        `toml:"block_device_cache_noflush"`
+	BlockDeviceCacheNoflush        bool            `toml:"block_device_cache_noflush"`
-	EnableVhostUserStore           bool        `toml:"enable_vhost_user_store"`
+	EnableVhostUserStore           bool            `toml:"enable_vhost_user_store"`
-	VhostUserDeviceReconnect       uint32      `toml:"vhost_user_reconnect_timeout_sec"`
+	VhostUserDeviceReconnect       uint32          `toml:"vhost_user_reconnect_timeout_sec"`
-	DisableBlockDeviceUse          bool        `toml:"disable_block_device_use"`
+	DisableBlockDeviceUse          bool            `toml:"disable_block_device_use"`
-	MemPrealloc                    bool        `toml:"enable_mem_prealloc"`
+	MemPrealloc                    bool            `toml:"enable_mem_prealloc"`
-	HugePages                      bool        `toml:"enable_hugepages"`
+	HugePages                      bool            `toml:"enable_hugepages"`
-	VirtioMem                      bool        `toml:"enable_virtio_mem"`
+	VirtioMem                      bool            `toml:"enable_virtio_mem"`
-	IOMMU                          bool        `toml:"enable_iommu"`
+	IOMMU                          bool            `toml:"enable_iommu"`
-	IOMMUPlatform                  bool        `toml:"enable_iommu_platform"`
+	IOMMUPlatform                  bool            `toml:"enable_iommu_platform"`
-	Debug                          bool        `toml:"enable_debug"`
+	Debug                          bool            `toml:"enable_debug"`
-	DisableNestingChecks           bool        `toml:"disable_nesting_checks"`
+	DisableNestingChecks           bool            `toml:"disable_nesting_checks"`
-	EnableIOThreads                bool        `toml:"enable_iothreads"`
+	EnableIOThreads                bool            `toml:"enable_iothreads"`
-	DisableImageNvdimm             bool        `toml:"disable_image_nvdimm"`
+	DisableImageNvdimm             bool            `toml:"disable_image_nvdimm"`
-	HotplugVFIOOnRootBus           bool        `toml:"hotplug_vfio_on_root_bus"`
+	HotplugVFIOOnRootBus           bool            `toml:"hotplug_vfio_on_root_bus"`
-	HotPlugVFIO                    hv.PCIePort `toml:"hot_plug_vfio"`
+	HotPlugVFIO                    config.PCIePort `toml:"hot_plug_vfio"`
-	ColdPlugVFIO                   hv.PCIePort `toml:"cold_plug_vfio"`
+	ColdPlugVFIO                   config.PCIePort `toml:"cold_plug_vfio"`
-	DisableVhostNet                bool        `toml:"disable_vhost_net"`
+	DisableVhostNet                bool            `toml:"disable_vhost_net"`
-	GuestMemoryDumpPaging          bool        `toml:"guest_memory_dump_paging"`
+	GuestMemoryDumpPaging          bool            `toml:"guest_memory_dump_paging"`
-	ConfidentialGuest              bool        `toml:"confidential_guest"`
+	ConfidentialGuest              bool            `toml:"confidential_guest"`
-	SevSnpGuest                    bool        `toml:"sev_snp_guest"`
+	SevSnpGuest                    bool            `toml:"sev_snp_guest"`
-	GuestSwap                      bool        `toml:"enable_guest_swap"`
+	GuestSwap                      bool            `toml:"enable_guest_swap"`
-	Rootless                       bool        `toml:"rootless"`
+	Rootless                       bool            `toml:"rootless"`
-	DisableSeccomp                 bool        `toml:"disable_seccomp"`
+	DisableSeccomp                 bool            `toml:"disable_seccomp"`
-	DisableSeLinux                 bool        `toml:"disable_selinux"`
+	DisableSeLinux                 bool            `toml:"disable_selinux"`
-	DisableGuestSeLinux            bool        `toml:"disable_guest_selinux"`
+	DisableGuestSeLinux            bool            `toml:"disable_guest_selinux"`
-	LegacySerial                   bool        `toml:"use_legacy_serial"`
+	LegacySerial                   bool            `toml:"use_legacy_serial"`
 }
 
 type runtime struct {
@@ -289,13 +288,13 @@ func (h hypervisor) firmware() (string, error) {
 	return ResolvePath(p)
 }
 
-func (h hypervisor) coldPlugVFIO() hv.PCIePort {
+func (h hypervisor) coldPlugVFIO() config.PCIePort {
 	if h.ColdPlugVFIO == "" {
 		return defaultColdPlugVFIO
 	}
 	return h.ColdPlugVFIO
 }
-func (h hypervisor) hotPlugVFIO() hv.PCIePort {
+func (h hypervisor) hotPlugVFIO() config.PCIePort {
 	if h.HotPlugVFIO == "" {
 		return defaultHotPlugVFIO
 	}
@@ -1689,29 +1688,29 @@ func checkConfig(config oci.RuntimeConfig) error {
 // checkPCIeConfig ensures the PCIe configuration is valid.
 // Only allow one of the following settings for cold-plug:
 // no-port, root-port, switch-port
-func checkPCIeConfig(coldPlug hv.PCIePort, hotPlug hv.PCIePort, machineType string) error {
+func checkPCIeConfig(coldPlug config.PCIePort, hotPlug config.PCIePort, machineType string) error {
 	// Currently only QEMU q35 supports advanced PCIe topologies
 	// firecracker, dragonball do not have right now any PCIe support
 	if machineType != "q35" {
 		return nil
 	}
 
-	if coldPlug != hv.NoPort && hotPlug != hv.NoPort {
+	if coldPlug != config.NoPort && hotPlug != config.NoPort {
 		return fmt.Errorf("invalid hot-plug=%s and cold-plug=%s settings, only one of them can be set", coldPlug, hotPlug)
 	}
-	var port hv.PCIePort
+	var port config.PCIePort
-	if coldPlug != hv.NoPort {
+	if coldPlug != config.NoPort {
 		port = coldPlug
 	}
-	if hotPlug != hv.NoPort {
+	if hotPlug != config.NoPort {
 		port = hotPlug
 	}
-	if port == hv.NoPort || port == hv.BridgePort || port == hv.RootPort || port == hv.SwitchPort {
+	if port == config.NoPort || port == config.BridgePort || port == config.RootPort || port == config.SwitchPort {
 		return nil
 	}
 
 	return fmt.Errorf("invalid vfio_port=%s setting, allowed values %s, %s, %s, %s",
-		coldPlug, hv.NoPort, hv.BridgePort, hv.RootPort, hv.SwitchPort)
+		coldPlug, config.NoPort, config.BridgePort, config.RootPort, config.SwitchPort)
 }
 
 // checkNetNsConfig performs sanity checks on disable_new_netns config.

src/runtime/pkg/katautils/config_test.go

@@ -18,8 +18,8 @@ import (
 	"syscall"
 	"testing"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
-	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
@@ -63,16 +63,16 @@ func createConfig(configPath string, fileData string) error {
 
 // createAllRuntimeConfigFiles creates all files necessary to call
 // loadConfiguration().
-func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConfig, err error) {
+func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntimeConfig, err error) {
 	if dir == "" {
-		return config, fmt.Errorf("BUG: need directory")
+		return testConfig, fmt.Errorf("BUG: need directory")
 	}
 
 	if hypervisor == "" {
-		return config, fmt.Errorf("BUG: need hypervisor")
+		return testConfig, fmt.Errorf("BUG: need hypervisor")
 	}
-	var hotPlugVFIO hv.PCIePort
+	var hotPlugVFIO config.PCIePort
-	var coldPlugVFIO hv.PCIePort
+	var coldPlugVFIO config.PCIePort
 	hypervisorPath := path.Join(dir, "hypervisor")
 	kernelPath := path.Join(dir, "kernel")
 	kernelParams := "foo=bar xyz"
@@ -88,8 +88,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 	hotplugVFIOOnRootBus := true
 	pcieRootPort := uint32(2)
 	pcieSwitchPort := uint32(3)
-	hotPlugVFIO = hv.BridgePort
+	hotPlugVFIO = config.BridgePort
-	coldPlugVFIO = hv.RootPort
+	coldPlugVFIO = config.RootPort
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
 	virtioFSdaemon := path.Join(dir, "virtiofsd")
@@ -139,7 +139,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 	configPath := path.Join(dir, "runtime.toml")
 	err = createConfig(configPath, runtimeConfigFileData)
 	if err != nil {
-		return config, err
+		return testConfig, err
 	}
 
 	configPathLink := path.Join(filepath.Dir(configPath), "link-to-configuration.toml")
@@ -147,7 +147,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 	// create a link to the config file
 	err = syscall.Symlink(configPath, configPathLink)
 	if err != nil {
-		return config, err
+		return testConfig, err
 	}
 
 	files := []string{hypervisorPath, kernelPath, imagePath}
@@ -156,7 +156,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		// create the resource (which must be >0 bytes)
 		err := WriteFile(file, "foo", testFileMode)
 		if err != nil {
-			return config, err
+			return testConfig, err
 		}
 	}
 
@@ -223,10 +223,10 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 
 	err = SetKernelParams(&runtimeConfig)
 	if err != nil {
-		return config, err
+		return testConfig, err
 	}
 
-	config = testRuntimeConfig{
+	rtimeConfig := testRuntimeConfig{
 		RuntimeConfig:     runtimeConfig,
 		RuntimeConfigFile: configPath,
 		ConfigPath:        configPath,
@@ -235,7 +235,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		LogPath:           logPath,
 	}
 
-	return config, nil
+	return rtimeConfig, nil
 }
 
 // testLoadConfiguration accepts an optional function that can be used
@@ -610,7 +610,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 
 func TestNewQemuHypervisorConfig(t *testing.T) {
 	dir := t.TempDir()
-	var coldPlugVFIO hv.PCIePort
+	var coldPlugVFIO config.PCIePort
 	hypervisorPath := path.Join(dir, "hypervisor")
 	kernelPath := path.Join(dir, "kernel")
 	imagePath := path.Join(dir, "image")
@@ -619,7 +619,7 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
 	pcieRootPort := uint32(2)
-	coldPlugVFIO = hv.RootPort
+	coldPlugVFIO = config.RootPort
 	orgVHostVSockDevicePath := utils.VHostVSockDevicePath
 	blockDeviceAIO := "io_uring"
 	defer func() {

src/runtime/virtcontainers/hypervisor.go

@@ -511,11 +511,11 @@ type HypervisorConfig struct {
 
 	// HotplugVFIO is used to indicate if devices need to be hotplugged on the
 	// root port or a switch
-	HotPlugVFIO hv.PCIePort
+	HotPlugVFIO config.PCIePort
 
 	// ColdPlugVFIO is used to indicate if devices need to be coldplugged on the
 	// root port, switch or no port
-	ColdPlugVFIO hv.PCIePort
+	ColdPlugVFIO config.PCIePort
 
 	// PCIeRootPort is used to indicate the number of PCIe Root Port devices
 	// The PCIe Root Port device is used to hot-plug the PCIe device

src/runtime/virtcontainers/kata_agent_test.go

@@ -1048,16 +1048,16 @@ func TestKataAgentKernelParams(t *testing.T) {
 		{true, true, 0, []Param{debugParam, traceParam}},
 
 		// pipesize
-		{false, false, 0, []Param{containerPipeSizeParam}},
+		{false, false, 2097152, []Param{containerPipeSizeParam}},
 
 		// Debug + pipesize
-		{true, false, 0, []Param{debugParam, containerPipeSizeParam}},
+		{true, false, 2097152, []Param{debugParam, containerPipeSizeParam}},
 
 		// Tracing + pipesize
-		{false, true, 0, []Param{traceParam, containerPipeSizeParam}},
+		{false, true, 2097152, []Param{traceParam, containerPipeSizeParam}},
 
 		// Debug + Tracing + pipesize
-		{true, true, 0, []Param{debugParam, traceParam, containerPipeSizeParam}},
+		{true, true, 2097152, []Param{debugParam, traceParam, containerPipeSizeParam}},
 	}
 
 	for i, d := range data {

src/runtime/virtcontainers/nydusd_test.go

@@ -57,13 +57,13 @@ func TestNydusdStart(t *testing.T) {
 
 	// nolint: govet
 	tests := []struct {
-		fields  fields
 		name    string
+		fields  fields
 		wantErr bool
 	}{
-		{name: "empty config", fields: fields{}, wantErr: true},
+		{"empty config", fields{}, true},
-		{name: "directory source path not exist", fields: SourcePathNoExist, wantErr: true},
+		{"directory source path not exist", SourcePathNoExist, true},
-		{name: "valid config", fields: validConfig, wantErr: false},
+		{"valid config", validConfig, false},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

src/runtime/virtcontainers/persist/api/config.go

@@ -7,7 +7,7 @@
 package persistapi
 
 import (
-	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/opencontainers/runc/libcontainer/configs"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
@@ -205,11 +205,11 @@ type HypervisorConfig struct {
 
 	// HotPlugVFIO is used to indicate if devices need to be hotplugged on the
 	// root, switch, bridge or no-port
-	HotPlugVFIO hv.PCIePort
+	HotPlugVFIO config.PCIePort
 
 	// ColdPlugVFIO is used to indicate if devices need to be coldplugged on the
 	// root, bridge, switch or no-port
-	ColdPlugVFIO hv.PCIePort
+	ColdPlugVFIO config.PCIePort
 
 	// BootToBeTemplate used to indicate if the VM is created to be a template VM
 	BootToBeTemplate bool

src/runtime/virtcontainers/qemu.go

@@ -76,7 +76,7 @@ type qmpChannel struct {
 // QemuState keeps Qemu's state
 type QemuState struct {
 	UUID                 string
-	HotPlugVFIO          hv.PCIePort
+	HotPlugVFIO          config.PCIePort
 	Bridges              []types.Bridge
 	HotpluggedVCPUs      []hv.CPUDevice
 	HotpluggedMemory     int
@@ -84,8 +84,8 @@ type QemuState struct {
 	PCIeRootPort         int
 	PCIeSwitchPort       int
 	HotplugVFIOOnRootBus bool
-	HotplugVFIO          hv.PCIePort
+	HotplugVFIO          config.PCIePort
-	ColdPlugVFIO         hv.PCIePort
+	ColdPlugVFIO         config.PCIePort
 }
 
 // qemu is an Hypervisor interface implementation for the Linux qemu hypervisor.
@@ -748,7 +748,7 @@ func (q *qemu) checkBpfEnabled() {
 // only 16 ports possible.
 func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig *HypervisorConfig) error {
 	// If no-port set just return no need to add PCIe Root Port or PCIe Switches
-	if hypervisorConfig.HotPlugVFIO == hv.NoPort && hypervisorConfig.ColdPlugVFIO == hv.NoPort {
+	if hypervisorConfig.HotPlugVFIO == config.NoPort && hypervisorConfig.ColdPlugVFIO == config.NoPort {
 		return nil
 	}
 	// Add PCIe Root Port or PCIe Switches to the hypervisor
@@ -787,11 +787,14 @@ func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig
 			if drivers.IsPCIeDevice(vfioDevice.BDF) {
 				numOfPluggablePorts = numOfPluggablePorts + 1
 			}
-			// Reset the Rank, the vfio module is going to assign the correct one
+			// Reset the Rank and AllPCIeDevsthe vfio module is going
-			vfioDevice.Rank = -1
+			// to assign the correct one in the case of hot-plug
+			if hypervisorConfig.HotPlugVFIO != config.NoPort {
+				vfioDevice.Rank = -1
+				drivers.AllPCIeDevs = map[string]bool{}
+			}
 		}
 	}
-	drivers.AllPCIeDevs = make(map[string]bool)
 
 	// If number of PCIe root ports > 16 then bail out otherwise we may
 	// use up all slots or IO memory on the root bus and vfio-XXX-pci devices
@@ -815,10 +818,10 @@ func (q *qemu) createPCIeTopology(qemuConfig *govmmQemu.Config, hypervisorConfig
 		numOfPluggablePorts = hypervisorConfig.PCIeSwitchPort
 	}
 
-	if q.state.HotPlugVFIO == hv.RootPort || q.state.ColdPlugVFIO == hv.RootPort || q.state.HotplugVFIOOnRootBus {
+	if q.state.HotPlugVFIO == config.RootPort || q.state.ColdPlugVFIO == config.RootPort || q.state.HotplugVFIOOnRootBus {
 		qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, numOfPluggablePorts, memSize32bit, memSize64bit)
 	}
-	if q.state.HotPlugVFIO == hv.SwitchPort || q.state.ColdPlugVFIO == hv.SwitchPort {
+	if q.state.HotPlugVFIO == config.SwitchPort || q.state.ColdPlugVFIO == config.SwitchPort {
 		qemuConfig.Devices = q.arch.appendPCIeSwitchPortDevice(qemuConfig.Devices, numOfPluggablePorts, memSize32bit, memSize64bit)
 	}
 	return nil
@@ -1629,7 +1632,7 @@ func (q *qemu) hotplugAddVhostUserBlkDevice(ctx context.Context, vAttr *config.V
 		//Since the dev is the first and only one on this bus(root port), it should be 0.
 		addr := "00"
 
-		bridgeID := fmt.Sprintf("%s%d", pcieRootPortPrefix, len(drivers.AllPCIeDevs))
+		bridgeID := fmt.Sprintf("%s%d", config.PCIeRootPortPrefix, len(drivers.AllPCIeDevs))
 		drivers.AllPCIeDevs[devID] = true
 
 		bridgeQomPath := fmt.Sprintf("%s%s", qomPathPrefix, bridgeID)
@@ -1824,7 +1827,6 @@ func (q *qemu) hotplugVFIODeviceRootPort(ctx context.Context, device *config.VFI
 			q.state.HotPlugVFIO, q.state.PCIeRootPort, q.state.PCIeSwitchPort)
 	}
 
-	device.Bus = fmt.Sprintf("%s%d", pcieRootPortPrefix, device.Rank)
 	return q.executeVFIODeviceAdd(device)
 }
 
@@ -1837,7 +1839,6 @@ func (q *qemu) hotplugVFIODeviceSwitchPort(ctx context.Context, device *config.V
 			q.state.HotPlugVFIO, q.state.PCIeRootPort, q.state.PCIeSwitchPort)
 	}
 
-	device.Bus = fmt.Sprintf("%s%d", pcieSwitchDownstreamPortPrefix, device.Rank)
 	return q.executeVFIODeviceAdd(device)
 }
 
@@ -1904,9 +1905,9 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op
 			// In case HotplugVFIOOnRootBus is true, devices are hotplugged on the root bus
 			// for pc machine type instead of bridge. This is useful for devices that require
 			// a large PCI BAR which is a currently a limitation with PCI bridges.
-			if q.state.HotPlugVFIO == hv.RootPort || q.state.HotplugVFIOOnRootBus {
+			if q.state.HotPlugVFIO == config.RootPort || q.state.HotplugVFIOOnRootBus {
 				err = q.hotplugVFIODeviceRootPort(ctx, device)
-			} else if q.state.HotPlugVFIO == hv.SwitchPort {
+			} else if q.state.HotPlugVFIO == config.SwitchPort {
 				err = q.hotplugVFIODeviceSwitchPort(ctx, device)
 			} else {
 				err = q.hotplugVFIODeviceBridgePort(ctx, device)
@@ -2632,7 +2633,7 @@ func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machin
 	for i := uint32(0); i < number; i++ {
 		devices = append(devices,
 			govmmQemu.PCIeRootPortDevice{
-				ID:            fmt.Sprintf("%s%d", pcieRootPortPrefix, i),
+				ID:            fmt.Sprintf("%s%d", config.PCIeRootPortPrefix, i),
 				Bus:           bus,
 				Chassis:       chassis,
 				Slot:          strconv.FormatUint(uint64(i), 10),
@@ -2679,7 +2680,7 @@ func genericAppendPCIeSwitchPort(devices []govmmQemu.Device, number uint32, mach
 	// Using an own ID for the root port, so we do not clash with already
 	// existing root ports adding "s" for switch prefix
 	pcieRootPort := govmmQemu.PCIeRootPortDevice{
-		ID:            fmt.Sprintf("%s%s%d", pcieSwitchPrefix, pcieRootPortPrefix, 0),
+		ID:            fmt.Sprintf("%s%s%d", config.PCIeSwitchPortPrefix, config.PCIeRootPortPrefix, 0),
 		Bus:           defaultBridgeBus,
 		Chassis:       "0",
 		Slot:          strconv.FormatUint(uint64(0), 10),
@@ -2692,7 +2693,7 @@ func genericAppendPCIeSwitchPort(devices []govmmQemu.Device, number uint32, mach
 	devices = append(devices, pcieRootPort)
 
 	pcieSwitchUpstreamPort := govmmQemu.PCIeSwitchUpstreamPortDevice{
-		ID:  fmt.Sprintf("%s%d", pcieSwitchUpstreamPortPrefix, 0),
+		ID:  fmt.Sprintf("%s%d", config.PCIeSwitchUpstreamPortPrefix, 0),
 		Bus: pcieRootPort.ID,
 	}
 	devices = append(devices, pcieSwitchUpstreamPort)
@@ -2706,7 +2707,7 @@ func genericAppendPCIeSwitchPort(devices []govmmQemu.Device, number uint32, mach
 	for i := uint32(0); i < number; i++ {
 
 		pcieSwitchDownstreamPort := govmmQemu.PCIeSwitchDownstreamPortDevice{
-			ID:      fmt.Sprintf("%s%d", pcieSwitchDownstreamPortPrefix, i),
+			ID:      fmt.Sprintf("%s%d", config.PCIeSwitchhDownstreamPortPrefix, i),
 			Bus:     pcieSwitchUpstreamPort.ID,
 			Chassis: fmt.Sprintf("%d", nextChassis),
 			Slot:    strconv.FormatUint(uint64(i), 10),

src/runtime/virtcontainers/qemu_arch_base.go

@@ -180,18 +180,14 @@ type qemuArchBase struct {
 }
 
 const (
-	defaultCores                   uint32 = 1
+	defaultCores       uint32 = 1
-	defaultThreads                 uint32 = 1
+	defaultThreads     uint32 = 1
-	defaultCPUModel                       = "host"
+	defaultCPUModel           = "host"
-	defaultBridgeBus                      = "pcie.0"
+	defaultBridgeBus          = "pcie.0"
-	defaultPCBridgeBus                    = "pci.0"
+	defaultPCBridgeBus        = "pci.0"
-	maxDevIDSize                          = 31
+	maxDevIDSize              = 31
-	maxPCIeRootPort                       = 16 // Limitation from QEMU
+	maxPCIeRootPort           = 16 // Limitation from QEMU
-	maxPCIeSwitchPort                     = 16 // Limitation from QEMU
+	maxPCIeSwitchPort         = 16 // Limitation from QEMU
-	pcieRootPortPrefix                    = "rp"
-	pcieSwitchPrefix                      = "sw"
-	pcieSwitchUpstreamPortPrefix          = "swup"
-	pcieSwitchDownstreamPortPrefix        = "swdp"
 )
 
 // This is the PCI start address assigned to the first bridge that

src/runtime/virtcontainers/sandbox.go

@@ -32,7 +32,6 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
 	deviceManager "github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager"
-	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
@@ -616,10 +615,10 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 
 	// If we have a confidential guest we need to cold-plug the PCIe VFIO devices
 	// until we have TDISP/IDE PCIe support.
-	coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO != hv.NoPort)
+	coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO != config.NoPort)
 	// Aggregate all the containner devices for hot-plug and use them to dedcue
 	// the correct amount of ports to reserve for the hypervisor.
-	hotPlugVFIO := (sandboxConfig.HypervisorConfig.HotPlugVFIO != hv.NoPort)
+	hotPlugVFIO := (sandboxConfig.HypervisorConfig.HotPlugVFIO != config.NoPort)
 
 	var vfioHotPlugDevices []config.DeviceInfo
 	var vfioColdPlugDevices []config.DeviceInfo
@@ -629,9 +628,11 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 			isVFIO := deviceManager.IsVFIO(device.ContainerPath)
 			if hotPlugVFIO && isVFIO {
 				vfioHotPlugDevices = append(vfioHotPlugDevices, device)
+				sandboxConfig.Containers[cnt].DeviceInfos[dev].Port = sandboxConfig.HypervisorConfig.HotPlugVFIO
 			}
 			if coldPlugVFIO && isVFIO {
 				device.ColdPlug = true
+				device.Port = sandboxConfig.HypervisorConfig.ColdPlugVFIO
 				vfioColdPlugDevices = append(vfioColdPlugDevices, device)
 				// We need to remove the devices marked for cold-plug
 				// otherwise at the container level the kata-agent
@@ -639,6 +640,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 				infos := sandboxConfig.Containers[cnt].DeviceInfos
 				infos = append(infos[:dev], infos[dev+1:]...)
 				sandboxConfig.Containers[cnt].DeviceInfos = infos
+
 			}
 		}
 	}
@@ -2015,7 +2017,9 @@ func (s *Sandbox) AppendDevice(ctx context.Context, device api.Device) error {
 		return s.hypervisor.AddDevice(ctx, device.GetDeviceInfo().(*config.VhostUserDeviceAttrs), VhostuserDev)
 	case config.DeviceVFIO:
 		vfioDevs := device.GetDeviceInfo().([]*config.VFIODev)
+		s.Logger().Info("### vfioDevs: ", vfioDevs)
 		for _, d := range vfioDevs {
+			s.Logger().Info("### vfioDev: ", d)
 			return s.hypervisor.AddDevice(ctx, *d, VfioDev)
 		}
 	default: