mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-07-04 11:06:21 +00:00
vfio: Add configuration to support VFIO hotplug on root bus
We need this configuration due to a limitation in seabios firmware in handling hotplug for PCI devices with large BARS. Long term, this needs to be fixed in the firmware. Fixes #594 Signed-off-by: Archana Shinde <archana.m.shinde@intel.com>
This commit is contained in:
parent
dd2acd26eb
commit
31e2925a9a
4
Makefile
4
Makefile
@ -130,6 +130,7 @@ DEFENABLESWAP := false
|
|||||||
DEFENABLEDEBUG := false
|
DEFENABLEDEBUG := false
|
||||||
DEFDISABLENESTINGCHECKS := false
|
DEFDISABLENESTINGCHECKS := false
|
||||||
DEFMSIZE9P := 8192
|
DEFMSIZE9P := 8192
|
||||||
|
DEFHOTPLUGVFIOONROOTBUS := false
|
||||||
|
|
||||||
SED = sed
|
SED = sed
|
||||||
|
|
||||||
@ -202,6 +203,7 @@ USER_VARS += DEFENABLESWAP
|
|||||||
USER_VARS += DEFENABLEDEBUG
|
USER_VARS += DEFENABLEDEBUG
|
||||||
USER_VARS += DEFDISABLENESTINGCHECKS
|
USER_VARS += DEFDISABLENESTINGCHECKS
|
||||||
USER_VARS += DEFMSIZE9P
|
USER_VARS += DEFMSIZE9P
|
||||||
|
USER_VARS += DEFHOTPLUGVFIOONROOTBUS
|
||||||
|
|
||||||
V = @
|
V = @
|
||||||
Q = $(V:1=)
|
Q = $(V:1=)
|
||||||
@ -296,6 +298,7 @@ const defaultEnableSwap bool = $(DEFENABLESWAP)
|
|||||||
const defaultEnableDebug bool = $(DEFENABLEDEBUG)
|
const defaultEnableDebug bool = $(DEFENABLEDEBUG)
|
||||||
const defaultDisableNestingChecks bool = $(DEFDISABLENESTINGCHECKS)
|
const defaultDisableNestingChecks bool = $(DEFDISABLENESTINGCHECKS)
|
||||||
const defaultMsize9p uint32 = $(DEFMSIZE9P)
|
const defaultMsize9p uint32 = $(DEFMSIZE9P)
|
||||||
|
const defaultHotplugVFIOOnRootBus bool = $(DEFHOTPLUGVFIOONROOTBUS)
|
||||||
|
|
||||||
// Default config file used by stateless systems.
|
// Default config file used by stateless systems.
|
||||||
var defaultRuntimeConfiguration = "$(CONFIG_PATH)"
|
var defaultRuntimeConfiguration = "$(CONFIG_PATH)"
|
||||||
@ -382,6 +385,7 @@ $(GENERATED_FILES): %: %.in Makefile VERSION
|
|||||||
-e "s|@DEFENABLEDEBUG@|$(DEFENABLEDEBUG)|g" \
|
-e "s|@DEFENABLEDEBUG@|$(DEFENABLEDEBUG)|g" \
|
||||||
-e "s|@DEFDISABLENESTINGCHECKS@|$(DEFDISABLENESTINGCHECKS)|g" \
|
-e "s|@DEFDISABLENESTINGCHECKS@|$(DEFDISABLENESTINGCHECKS)|g" \
|
||||||
-e "s|@DEFMSIZE9P@|$(DEFMSIZE9P)|g" \
|
-e "s|@DEFMSIZE9P@|$(DEFMSIZE9P)|g" \
|
||||||
|
-e "s|@DEFHOTPLUGONROOTBUS@|$(DEFHOTPLUGVFIOONROOTBUS)|g" \
|
||||||
$< > $@
|
$< > $@
|
||||||
|
|
||||||
generate-config: $(CONFIG)
|
generate-config: $(CONFIG)
|
||||||
|
@ -79,12 +79,12 @@ type hypervisor struct {
|
|||||||
MachineAccelerators string `toml:"machine_accelerators"`
|
MachineAccelerators string `toml:"machine_accelerators"`
|
||||||
KernelParams string `toml:"kernel_params"`
|
KernelParams string `toml:"kernel_params"`
|
||||||
MachineType string `toml:"machine_type"`
|
MachineType string `toml:"machine_type"`
|
||||||
|
BlockDeviceDriver string `toml:"block_device_driver"`
|
||||||
DefaultVCPUs int32 `toml:"default_vcpus"`
|
DefaultVCPUs int32 `toml:"default_vcpus"`
|
||||||
DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"`
|
DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"`
|
||||||
DefaultMemSz uint32 `toml:"default_memory"`
|
DefaultMemSz uint32 `toml:"default_memory"`
|
||||||
DefaultBridges uint32 `toml:"default_bridges"`
|
DefaultBridges uint32 `toml:"default_bridges"`
|
||||||
Msize9p uint32 `toml:"msize_9p"`
|
Msize9p uint32 `toml:"msize_9p"`
|
||||||
BlockDeviceDriver string `toml:"block_device_driver"`
|
|
||||||
DisableBlockDeviceUse bool `toml:"disable_block_device_use"`
|
DisableBlockDeviceUse bool `toml:"disable_block_device_use"`
|
||||||
MemPrealloc bool `toml:"enable_mem_prealloc"`
|
MemPrealloc bool `toml:"enable_mem_prealloc"`
|
||||||
HugePages bool `toml:"enable_hugepages"`
|
HugePages bool `toml:"enable_hugepages"`
|
||||||
@ -93,6 +93,7 @@ type hypervisor struct {
|
|||||||
DisableNestingChecks bool `toml:"disable_nesting_checks"`
|
DisableNestingChecks bool `toml:"disable_nesting_checks"`
|
||||||
EnableIOThreads bool `toml:"enable_iothreads"`
|
EnableIOThreads bool `toml:"enable_iothreads"`
|
||||||
UseVSock bool `toml:"use_vsock"`
|
UseVSock bool `toml:"use_vsock"`
|
||||||
|
HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type proxy struct {
|
type proxy struct {
|
||||||
@ -373,6 +374,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
|
|||||||
EnableIOThreads: h.EnableIOThreads,
|
EnableIOThreads: h.EnableIOThreads,
|
||||||
Msize9p: h.msize9p(),
|
Msize9p: h.msize9p(),
|
||||||
UseVSock: useVSock,
|
UseVSock: useVSock,
|
||||||
|
HotplugVFIOOnRootBus: h.HotplugVFIOOnRootBus,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -489,6 +491,7 @@ func loadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat
|
|||||||
BlockDeviceDriver: defaultBlockDeviceDriver,
|
BlockDeviceDriver: defaultBlockDeviceDriver,
|
||||||
EnableIOThreads: defaultEnableIOThreads,
|
EnableIOThreads: defaultEnableIOThreads,
|
||||||
Msize9p: defaultMsize9p,
|
Msize9p: defaultMsize9p,
|
||||||
|
HotplugVFIOOnRootBus: defaultHotplugVFIOOnRootBus,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = config.InterNetworkModel.SetModel(defaultInterNetworkingModel)
|
err = config.InterNetworkModel.SetModel(defaultInterNetworkingModel)
|
||||||
|
@ -140,6 +140,13 @@ enable_iothreads = @DEFENABLEIOTHREADS@
|
|||||||
# Default false
|
# Default false
|
||||||
#use_vsock = true
|
#use_vsock = true
|
||||||
|
|
||||||
|
# VFIO devices are hotplugged on a bridge by default.
|
||||||
|
# Enable hotplugging on root bus. This may be required for devices with
|
||||||
|
# a large PCI bar, as this is a current limitation with hotplugging on
|
||||||
|
# a bridge. This value is valid for "pc" machine type.
|
||||||
|
# Default false
|
||||||
|
#hotplug_vfio_on_root_bus = true
|
||||||
|
|
||||||
[factory]
|
[factory]
|
||||||
# VM templating support. Once enabled, new VMs are created from template
|
# VM templating support. Once enabled, new VMs are created from template
|
||||||
# using vm cloning. They will share the same initial kernel, initramfs and
|
# using vm cloning. They will share the same initial kernel, initramfs and
|
||||||
|
@ -33,7 +33,7 @@ type testRuntimeConfig struct {
|
|||||||
LogPath string
|
LogPath string
|
||||||
}
|
}
|
||||||
|
|
||||||
func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath, kernelParams, machineType, shimPath, proxyPath, logPath string, disableBlock bool, blockDeviceDriver string, enableIOThreads bool) string {
|
func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath, kernelParams, machineType, shimPath, proxyPath, logPath string, disableBlock bool, blockDeviceDriver string, enableIOThreads bool, hotplugVFIOOnRootBus bool) string {
|
||||||
return `
|
return `
|
||||||
# Runtime configuration file
|
# Runtime configuration file
|
||||||
|
|
||||||
@ -49,6 +49,7 @@ func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath
|
|||||||
default_memory = ` + strconv.FormatUint(uint64(defaultMemSize), 10) + `
|
default_memory = ` + strconv.FormatUint(uint64(defaultMemSize), 10) + `
|
||||||
disable_block_device_use = ` + strconv.FormatBool(disableBlock) + `
|
disable_block_device_use = ` + strconv.FormatBool(disableBlock) + `
|
||||||
enable_iothreads = ` + strconv.FormatBool(enableIOThreads) + `
|
enable_iothreads = ` + strconv.FormatBool(enableIOThreads) + `
|
||||||
|
hotplug_vfio_on_root_bus = ` + strconv.FormatBool(hotplugVFIOOnRootBus) + `
|
||||||
msize_9p = ` + strconv.FormatUint(uint64(defaultMsize9p), 10) + `
|
msize_9p = ` + strconv.FormatUint(uint64(defaultMsize9p), 10) + `
|
||||||
|
|
||||||
[proxy.kata]
|
[proxy.kata]
|
||||||
@ -97,8 +98,9 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
|
|||||||
disableBlockDevice := true
|
disableBlockDevice := true
|
||||||
blockDeviceDriver := "virtio-scsi"
|
blockDeviceDriver := "virtio-scsi"
|
||||||
enableIOThreads := true
|
enableIOThreads := true
|
||||||
|
hotplugVFIOOnRootBus := true
|
||||||
|
|
||||||
runtimeConfigFileData := makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath, kernelParams, machineType, shimPath, proxyPath, logPath, disableBlockDevice, blockDeviceDriver, enableIOThreads)
|
runtimeConfigFileData := makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath, kernelParams, machineType, shimPath, proxyPath, logPath, disableBlockDevice, blockDeviceDriver, enableIOThreads, hotplugVFIOOnRootBus)
|
||||||
|
|
||||||
configPath := path.Join(dir, "runtime.toml")
|
configPath := path.Join(dir, "runtime.toml")
|
||||||
err = createConfig(configPath, runtimeConfigFileData)
|
err = createConfig(configPath, runtimeConfigFileData)
|
||||||
@ -138,6 +140,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
|
|||||||
DefaultBridges: defaultBridgesCount,
|
DefaultBridges: defaultBridgesCount,
|
||||||
Mlock: !defaultEnableSwap,
|
Mlock: !defaultEnableSwap,
|
||||||
EnableIOThreads: enableIOThreads,
|
EnableIOThreads: enableIOThreads,
|
||||||
|
HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
|
||||||
Msize9p: defaultMsize9p,
|
Msize9p: defaultMsize9p,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -617,6 +620,7 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
|
|||||||
machineType := "machineType"
|
machineType := "machineType"
|
||||||
disableBlock := true
|
disableBlock := true
|
||||||
enableIOThreads := true
|
enableIOThreads := true
|
||||||
|
hotplugVFIOOnRootBus := true
|
||||||
orgVSockDevicePath := utils.VSockDevicePath
|
orgVSockDevicePath := utils.VSockDevicePath
|
||||||
orgVHostVSockDevicePath := utils.VHostVSockDevicePath
|
orgVHostVSockDevicePath := utils.VHostVSockDevicePath
|
||||||
defer func() {
|
defer func() {
|
||||||
@ -633,6 +637,7 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
|
|||||||
MachineType: machineType,
|
MachineType: machineType,
|
||||||
DisableBlockDeviceUse: disableBlock,
|
DisableBlockDeviceUse: disableBlock,
|
||||||
EnableIOThreads: enableIOThreads,
|
EnableIOThreads: enableIOThreads,
|
||||||
|
HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
|
||||||
UseVSock: true,
|
UseVSock: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -688,6 +693,9 @@ func TestNewQemuHypervisorConfig(t *testing.T) {
|
|||||||
t.Errorf("Expected value for enable IOThreads %v, got %v", enableIOThreads, config.EnableIOThreads)
|
t.Errorf("Expected value for enable IOThreads %v, got %v", enableIOThreads, config.EnableIOThreads)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if config.HotplugVFIOOnRootBus != hotplugVFIOOnRootBus {
|
||||||
|
t.Errorf("Expected value for HotplugVFIOOnRootBus %v, got %v", hotplugVFIOOnRootBus, config.HotplugVFIOOnRootBus)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewQemuHypervisorConfigImageAndInitrd(t *testing.T) {
|
func TestNewQemuHypervisorConfigImageAndInitrd(t *testing.T) {
|
||||||
@ -710,6 +718,7 @@ func TestNewQemuHypervisorConfigImageAndInitrd(t *testing.T) {
|
|||||||
machineType := "machineType"
|
machineType := "machineType"
|
||||||
disableBlock := true
|
disableBlock := true
|
||||||
enableIOThreads := true
|
enableIOThreads := true
|
||||||
|
hotplugVFIOOnRootBus := true
|
||||||
|
|
||||||
hypervisor := hypervisor{
|
hypervisor := hypervisor{
|
||||||
Path: hypervisorPath,
|
Path: hypervisorPath,
|
||||||
@ -719,6 +728,7 @@ func TestNewQemuHypervisorConfigImageAndInitrd(t *testing.T) {
|
|||||||
MachineType: machineType,
|
MachineType: machineType,
|
||||||
DisableBlockDeviceUse: disableBlock,
|
DisableBlockDeviceUse: disableBlock,
|
||||||
EnableIOThreads: enableIOThreads,
|
EnableIOThreads: enableIOThreads,
|
||||||
|
HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = newQemuHypervisorConfig(hypervisor)
|
_, err = newQemuHypervisorConfig(hypervisor)
|
||||||
|
@ -63,6 +63,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
|
|||||||
disableBlock := true
|
disableBlock := true
|
||||||
blockStorageDriver := "virtio-scsi"
|
blockStorageDriver := "virtio-scsi"
|
||||||
enableIOThreads := true
|
enableIOThreads := true
|
||||||
|
hotplugVFIOOnRootBus := true
|
||||||
|
|
||||||
// override
|
// override
|
||||||
defaultProxyPath = proxyPath
|
defaultProxyPath = proxyPath
|
||||||
@ -108,6 +109,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
|
|||||||
disableBlock,
|
disableBlock,
|
||||||
blockStorageDriver,
|
blockStorageDriver,
|
||||||
enableIOThreads,
|
enableIOThreads,
|
||||||
|
hotplugVFIOOnRootBus,
|
||||||
)
|
)
|
||||||
|
|
||||||
configFile = path.Join(prefixDir, "runtime.toml")
|
configFile = path.Join(prefixDir, "runtime.toml")
|
||||||
|
@ -223,6 +223,10 @@ type HypervisorConfig struct {
|
|||||||
// UseVSock use a vsock for agent communication
|
// UseVSock use a vsock for agent communication
|
||||||
UseVSock bool
|
UseVSock bool
|
||||||
|
|
||||||
|
// HotplugVFIOOnRootBus is used to indicate if devices need to be hotplugged on the
|
||||||
|
// root bus instead of a bridge.
|
||||||
|
HotplugVFIOOnRootBus bool
|
||||||
|
|
||||||
// BootToBeTemplate used to indicate if the VM is created to be a template VM
|
// BootToBeTemplate used to indicate if the VM is created to be a template VM
|
||||||
BootToBeTemplate bool
|
BootToBeTemplate bool
|
||||||
|
|
||||||
|
@ -40,9 +40,10 @@ type CPUDevice struct {
|
|||||||
type QemuState struct {
|
type QemuState struct {
|
||||||
Bridges []Bridge
|
Bridges []Bridge
|
||||||
// HotpluggedCPUs is the list of CPUs that were hot-added
|
// HotpluggedCPUs is the list of CPUs that were hot-added
|
||||||
HotpluggedVCPUs []CPUDevice
|
HotpluggedVCPUs []CPUDevice
|
||||||
HotpluggedMemory int
|
HotpluggedMemory int
|
||||||
UUID string
|
UUID string
|
||||||
|
HotplugVFIOOnRootBus bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// qemu is an Hypervisor interface implementation for the Linux qemu hypervisor.
|
// qemu is an Hypervisor interface implementation for the Linux qemu hypervisor.
|
||||||
@ -195,6 +196,8 @@ func (q *qemu) init(id string, hypervisorConfig *HypervisorConfig, vmConfig Reso
|
|||||||
q.Logger().Debug("Creating UUID")
|
q.Logger().Debug("Creating UUID")
|
||||||
q.state.UUID = uuid.Generate().String()
|
q.state.UUID = uuid.Generate().String()
|
||||||
|
|
||||||
|
q.state.HotplugVFIOOnRootBus = q.config.HotplugVFIOOnRootBus
|
||||||
|
|
||||||
// The path might already exist, but in case of VM templating,
|
// The path might already exist, but in case of VM templating,
|
||||||
// we have to create it since the sandbox has not created it yet.
|
// we have to create it since the sandbox has not created it yet.
|
||||||
if err = os.MkdirAll(filepath.Join(runStoragePath, id), dirMode); err != nil {
|
if err = os.MkdirAll(filepath.Join(runStoragePath, id), dirMode); err != nil {
|
||||||
@ -736,6 +739,13 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error {
|
|||||||
devID := device.ID
|
devID := device.ID
|
||||||
|
|
||||||
if op == addDevice {
|
if op == addDevice {
|
||||||
|
// In case HotplugVFIOOnRootBus is true, devices are hotplugged on the root bus
|
||||||
|
// for pc machine type instead of bridge. This is useful for devices that require
|
||||||
|
// a large PCI BAR which is a currently a limitation with PCI bridges.
|
||||||
|
if q.state.HotplugVFIOOnRootBus {
|
||||||
|
return q.qmpMonitorCh.qmp.ExecuteVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF)
|
||||||
|
}
|
||||||
|
|
||||||
addr, bridge, err := q.addDeviceToBridge(devID)
|
addr, bridge, err := q.addDeviceToBridge(devID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@ -745,8 +755,10 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if err := q.removeDeviceFromBridge(devID); err != nil {
|
if !q.state.HotplugVFIOOnRootBus {
|
||||||
return err
|
if err := q.removeDeviceFromBridge(devID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil {
|
if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil {
|
||||||
|
Loading…
Reference in New Issue
Block a user