This commit is contained in:
Zhang Chen 2025-08-12 10:14:00 +00:00 committed by GitHub
commit c5221ca4c9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 173 additions and 62 deletions

View File

@ -260,6 +260,7 @@ DEFVIRTIOFSQUEUESIZE ?= 1024
# Make sure you quote args. # Make sure you quote args.
DEFVIRTIOFSEXTRAARGS ?= [\"--thread-pool-size=1\", \"--announce-submounts\"] DEFVIRTIOFSEXTRAARGS ?= [\"--thread-pool-size=1\", \"--announce-submounts\"]
DEFENABLEIOTHREADS := false DEFENABLEIOTHREADS := false
DEFINDEPIOTHREADS := 0
DEFENABLEVHOSTUSERSTORE := false DEFENABLEVHOSTUSERSTORE := false
DEFVHOSTUSERSTOREPATH := $(PKGRUNDIR)/vhost-user DEFVHOSTUSERSTOREPATH := $(PKGRUNDIR)/vhost-user
DEFVALIDVHOSTUSERSTOREPATHS := [\"$(DEFVHOSTUSERSTOREPATH)\"] DEFVALIDVHOSTUSERSTOREPATHS := [\"$(DEFVHOSTUSERSTOREPATH)\"]
@ -731,6 +732,7 @@ USER_VARS += DEFVIRTIOFSEXTRAARGS
USER_VARS += DEFENABLEANNOTATIONS USER_VARS += DEFENABLEANNOTATIONS
USER_VARS += DEFENABLEANNOTATIONSTEE USER_VARS += DEFENABLEANNOTATIONSTEE
USER_VARS += DEFENABLEIOTHREADS USER_VARS += DEFENABLEIOTHREADS
USER_VARS += DEFINDEPIOTHREADS
USER_VARS += DEFSECCOMPSANDBOXPARAM USER_VARS += DEFSECCOMPSANDBOXPARAM
USER_VARS += DEFENABLEVHOSTUSERSTORE USER_VARS += DEFENABLEVHOSTUSERSTORE
USER_VARS += DEFVHOSTUSERSTOREPATH USER_VARS += DEFVHOSTUSERSTOREPATH

View File

@ -259,11 +259,15 @@ block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
#block_device_cache_noflush = true #block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be # Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented # handled in a separate IO thread. This is currently implemented
# for SCSI. # for virtio-scsi and virtio-blk.
# #
enable_iothreads = @DEFENABLEIOTHREADS@ enable_iothreads = @DEFENABLEIOTHREADS@
# Independent IOThreads enables IO to be processed in a separate thread, it is
# for QEMU hotplug device attach to iothread, like virtio-blk.
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false # Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density # Enabling this will result in lower container density
# as all of the memory will be allocated and locked # as all of the memory will be allocated and locked

View File

@ -268,11 +268,15 @@ block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
#block_device_cache_noflush = true #block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be # Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented # handled in a separate IO thread. This is currently implemented
# for SCSI. # for virtio-scsi and virtio-blk.
# #
enable_iothreads = @DEFENABLEIOTHREADS@ enable_iothreads = @DEFENABLEIOTHREADS@
# Independent IOThreads enables IO to be processed in a separate thread, it is
# for QEMU hotplug device attach to iothread, like virtio-blk.
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false # Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density # Enabling this will result in lower container density
# as all of the memory will be allocated and locked # as all of the memory will be allocated and locked

View File

@ -252,11 +252,15 @@ block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
#block_device_cache_noflush = true #block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be # Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented # handled in a separate IO thread. This is currently implemented
# for SCSI. # for virtio-scsi and virtio-blk.
# #
enable_iothreads = @DEFENABLEIOTHREADS@ enable_iothreads = @DEFENABLEIOTHREADS@
# Independent IOThreads enables IO to be processed in a separate thread, it is
# for QEMU hotplug device attach to iothread, like virtio-blk.
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false # Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density # Enabling this will result in lower container density
# as all of the memory will be allocated and locked # as all of the memory will be allocated and locked

View File

@ -257,11 +257,15 @@ block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
#block_device_cache_noflush = true #block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be # Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented # handled in a separate IO thread. This is currently implemented
# for SCSI. # for virtio-scsi and virtio-blk.
# #
enable_iothreads = @DEFENABLEIOTHREADS@ enable_iothreads = @DEFENABLEIOTHREADS@
# Independent IOThreads enables IO to be processed in a separate thread, it is
# for QEMU hotplug device attach to iothread, like virtio-blk.
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false # Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density # Enabling this will result in lower container density
# as all of the memory will be allocated and locked # as all of the memory will be allocated and locked

View File

@ -243,11 +243,15 @@ block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
#block_device_cache_noflush = true #block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be # Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented # handled in a separate IO thread. This is currently implemented
# for SCSI. # for virtio-scsi and virtio-blk.
# #
enable_iothreads = @DEFENABLEIOTHREADS@ enable_iothreads = @DEFENABLEIOTHREADS@
# Independent IOThreads enables IO to be processed in a separate thread, it is
# for QEMU hotplug device attach to iothread, like virtio-blk.
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false # Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density # Enabling this will result in lower container density
# as all of the memory will be allocated and locked # as all of the memory will be allocated and locked

View File

@ -268,11 +268,15 @@ block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
#block_device_cache_noflush = true #block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be # Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented # handled in a separate IO thread. This is currently implemented
# for SCSI. # for virtio-scsi and virtio-blk.
# #
enable_iothreads = @DEFENABLEIOTHREADS@ enable_iothreads = @DEFENABLEIOTHREADS@
# Independent IOThreads enables IO to be processed in a separate thread, it is
# for QEMU hotplug device attach to iothread, like virtio-blk.
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false # Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density # Enabling this will result in lower container density
# as all of the memory will be allocated and locked # as all of the memory will be allocated and locked

View File

@ -253,11 +253,15 @@ block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
#block_device_cache_noflush = true #block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be # Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented # handled in a separate IO thread. This is currently implemented
# for SCSI. # for virtio-scsi and virtio-blk.
# #
enable_iothreads = @DEFENABLEIOTHREADS@ enable_iothreads = @DEFENABLEIOTHREADS@
# Independent IOThreads enables IO to be processed in a separate thread, it is
# for QEMU hotplug device attach to iothread, like virtio-blk.
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false # Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density # Enabling this will result in lower container density
# as all of the memory will be allocated and locked # as all of the memory will be allocated and locked

View File

@ -258,11 +258,15 @@ block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
#block_device_cache_noflush = true #block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be # Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented # handled in a separate IO thread. This is currently implemented
# for SCSI. # for virtio-scsi and virtio-blk.
# #
enable_iothreads = @DEFENABLEIOTHREADS@ enable_iothreads = @DEFENABLEIOTHREADS@
# Independent IOThreads enables IO to be processed in a separate thread, it is
# for QEMU hotplug device attach to iothread, like virtio-blk.
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false # Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density # Enabling this will result in lower container density
# as all of the memory will be allocated and locked # as all of the memory will be allocated and locked

View File

@ -1102,7 +1102,7 @@ func (q *QMP) ExecuteDeviceDel(ctx context.Context, devID string) error {
// disableModern indicates if virtio version 1.0 should be replaced by the // disableModern indicates if virtio version 1.0 should be replaced by the
// former version 0.9, as there is a KVM bug that occurs when using virtio // former version 0.9, as there is a KVM bug that occurs when using virtio
// 1.0 in nested environments. // 1.0 in nested environments.
func (q *QMP) ExecutePCIDeviceAdd(ctx context.Context, blockdevID, devID, driver, addr, bus, romfile string, queues int, shared, disableModern bool) error { func (q *QMP) ExecutePCIDeviceAdd(ctx context.Context, blockdevID, devID, driver, addr, bus, romfile string, queues int, shared, disableModern bool, iothreadID string) error {
args := map[string]interface{}{ args := map[string]interface{}{
"id": devID, "id": devID,
"driver": driver, "driver": driver,
@ -1129,6 +1129,10 @@ func (q *QMP) ExecutePCIDeviceAdd(ctx context.Context, blockdevID, devID, driver
} }
} }
if iothreadID != "" {
args["iothread"] = iothreadID
}
return q.executeCommand(ctx, "device_add", args, nil) return q.executeCommand(ctx, "device_add", args, nil)
} }

View File

@ -1066,7 +1066,7 @@ func TestQMPPCIDeviceAdd(t *testing.T) {
blockdevID := fmt.Sprintf("drive_%s", volumeUUID) blockdevID := fmt.Sprintf("drive_%s", volumeUUID)
devID := fmt.Sprintf("device_%s", volumeUUID) devID := fmt.Sprintf("device_%s", volumeUUID)
err := q.ExecutePCIDeviceAdd(context.Background(), blockdevID, devID, err := q.ExecutePCIDeviceAdd(context.Background(), blockdevID, devID,
"virtio-blk-pci", "0x1", "", "", 1, true, false) "virtio-blk-pci", "0x1", "", "", 1, true, false, "")
if err != nil { if err != nil {
t.Fatalf("Unexpected error %v", err) t.Fatalf("Unexpected error %v", err)
} }

View File

@ -207,41 +207,42 @@ const (
) )
type RuntimeConfigOptions struct { type RuntimeConfigOptions struct {
Hypervisor string Hypervisor string
HypervisorPath string HypervisorPath string
DefaultGuestHookPath string DefaultGuestHookPath string
KernelPath string KernelPath string
ImagePath string ImagePath string
RootfsType string RootfsType string
KernelParams string KernelParams string
MachineType string MachineType string
LogPath string LogPath string
BlockDeviceDriver string BlockDeviceDriver string
BlockDeviceAIO string BlockDeviceAIO string
SharedFS string SharedFS string
VirtioFSDaemon string VirtioFSDaemon string
JaegerEndpoint string JaegerEndpoint string
JaegerUser string JaegerUser string
JaegerPassword string JaegerPassword string
PFlash []string PFlash []string
HotPlugVFIO config.PCIePort HotPlugVFIO config.PCIePort
ColdPlugVFIO config.PCIePort ColdPlugVFIO config.PCIePort
PCIeRootPort uint32 PCIeRootPort uint32
PCIeSwitchPort uint32 PCIeSwitchPort uint32
DefaultVCPUCount uint32 DefaultVCPUCount uint32
DefaultMaxVCPUCount uint32 DefaultMaxVCPUCount uint32
DefaultMemSize uint32 DefaultMemSize uint32
DefaultMaxMemorySize uint64 DefaultMaxMemorySize uint64
DefaultMsize9p uint32 DefaultMsize9p uint32
DisableBlock bool DefaultIndepIOThreads uint32
EnableIOThreads bool DisableBlock bool
DisableNewNetNs bool EnableIOThreads bool
HypervisorDebug bool DisableNewNetNs bool
RuntimeDebug bool HypervisorDebug bool
RuntimeTrace bool RuntimeDebug bool
AgentDebug bool RuntimeTrace bool
AgentTrace bool AgentDebug bool
EnablePprof bool AgentTrace bool
EnablePprof bool
} }
// ContainerIDTestDataType is a type used to test Container and Sandbox ID's. // ContainerIDTestDataType is a type used to test Container and Sandbox ID's.
@ -318,6 +319,7 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
default_memory = ` + strconv.FormatUint(uint64(config.DefaultMemSize), 10) + ` default_memory = ` + strconv.FormatUint(uint64(config.DefaultMemSize), 10) + `
disable_block_device_use = ` + strconv.FormatBool(config.DisableBlock) + ` disable_block_device_use = ` + strconv.FormatBool(config.DisableBlock) + `
enable_iothreads = ` + strconv.FormatBool(config.EnableIOThreads) + ` enable_iothreads = ` + strconv.FormatBool(config.EnableIOThreads) + `
indep_iothreads = ` + strconv.FormatUint(uint64(config.DefaultIndepIOThreads), 10) + `
cold_plug_vfio = "` + config.ColdPlugVFIO.String() + `" cold_plug_vfio = "` + config.ColdPlugVFIO.String() + `"
hot_plug_vfio = "` + config.HotPlugVFIO.String() + `" hot_plug_vfio = "` + config.HotPlugVFIO.String() + `"
pcie_root_port = ` + strconv.FormatUint(uint64(config.PCIeRootPort), 10) + ` pcie_root_port = ` + strconv.FormatUint(uint64(config.PCIeRootPort), 10) + `

View File

@ -75,6 +75,7 @@ const defaultBlockDeviceCacheSet bool = false
const defaultBlockDeviceCacheDirect bool = false const defaultBlockDeviceCacheDirect bool = false
const defaultBlockDeviceCacheNoflush bool = false const defaultBlockDeviceCacheNoflush bool = false
const defaultEnableIOThreads bool = false const defaultEnableIOThreads bool = false
const defaultIndepIOThreads uint32 = 0
const defaultEnableMemPrealloc bool = false const defaultEnableMemPrealloc bool = false
const defaultEnableReclaimGuestFreedMemory bool = false const defaultEnableReclaimGuestFreedMemory bool = false
const defaultEnableHugePages bool = false const defaultEnableHugePages bool = false

View File

@ -155,6 +155,7 @@ type hypervisor struct {
Debug bool `toml:"enable_debug"` Debug bool `toml:"enable_debug"`
DisableNestingChecks bool `toml:"disable_nesting_checks"` DisableNestingChecks bool `toml:"disable_nesting_checks"`
EnableIOThreads bool `toml:"enable_iothreads"` EnableIOThreads bool `toml:"enable_iothreads"`
IndepIOThreads uint32 `toml:"indep_iothreads"`
DisableImageNvdimm bool `toml:"disable_image_nvdimm"` DisableImageNvdimm bool `toml:"disable_image_nvdimm"`
HotPlugVFIO config.PCIePort `toml:"hot_plug_vfio"` HotPlugVFIO config.PCIePort `toml:"hot_plug_vfio"`
ColdPlugVFIO config.PCIePort `toml:"cold_plug_vfio"` ColdPlugVFIO config.PCIePort `toml:"cold_plug_vfio"`
@ -614,6 +615,14 @@ func (h hypervisor) msize9p() uint32 {
return h.Msize9p return h.Msize9p
} }
func (h hypervisor) indepiothreads() uint32 {
if h.IndepIOThreads == 0 {
return defaultIndepIOThreads
}
return h.IndepIOThreads
}
func (h hypervisor) guestHookPath() string { func (h hypervisor) guestHookPath() string {
if h.GuestHookPath == "" { if h.GuestHookPath == "" {
return defaultGuestHookPath return defaultGuestHookPath
@ -810,6 +819,7 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
DisableNestingChecks: h.DisableNestingChecks, DisableNestingChecks: h.DisableNestingChecks,
BlockDeviceDriver: blockDriver, BlockDeviceDriver: blockDriver,
EnableIOThreads: h.EnableIOThreads, EnableIOThreads: h.EnableIOThreads,
IndepIOThreads: h.indepiothreads(),
DisableVhostNet: true, // vhost-net backend is not supported in Firecracker DisableVhostNet: true, // vhost-net backend is not supported in Firecracker
GuestHookPath: h.guestHookPath(), GuestHookPath: h.guestHookPath(),
RxRateLimiterMaxRate: rxRateLimiterMaxRate, RxRateLimiterMaxRate: rxRateLimiterMaxRate,
@ -964,6 +974,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
BlockDeviceCacheDirect: h.BlockDeviceCacheDirect, BlockDeviceCacheDirect: h.BlockDeviceCacheDirect,
BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush, BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush,
EnableIOThreads: h.EnableIOThreads, EnableIOThreads: h.EnableIOThreads,
IndepIOThreads: h.indepiothreads(),
Msize9p: h.msize9p(), Msize9p: h.msize9p(),
DisableImageNvdimm: h.DisableImageNvdimm, DisableImageNvdimm: h.DisableImageNvdimm,
HotPlugVFIO: h.hotPlugVFIO(), HotPlugVFIO: h.hotPlugVFIO(),
@ -1094,6 +1105,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
BlockDeviceCacheSet: h.BlockDeviceCacheSet, BlockDeviceCacheSet: h.BlockDeviceCacheSet,
BlockDeviceCacheDirect: h.BlockDeviceCacheDirect, BlockDeviceCacheDirect: h.BlockDeviceCacheDirect,
EnableIOThreads: h.EnableIOThreads, EnableIOThreads: h.EnableIOThreads,
IndepIOThreads: h.indepiothreads(),
Msize9p: h.msize9p(), Msize9p: h.msize9p(),
DisableImageNvdimm: h.DisableImageNvdimm, DisableImageNvdimm: h.DisableImageNvdimm,
ColdPlugVFIO: h.coldPlugVFIO(), ColdPlugVFIO: h.coldPlugVFIO(),
@ -1452,6 +1464,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
BlockDeviceCacheDirect: defaultBlockDeviceCacheDirect, BlockDeviceCacheDirect: defaultBlockDeviceCacheDirect,
BlockDeviceCacheNoflush: defaultBlockDeviceCacheNoflush, BlockDeviceCacheNoflush: defaultBlockDeviceCacheNoflush,
EnableIOThreads: defaultEnableIOThreads, EnableIOThreads: defaultEnableIOThreads,
IndepIOThreads: defaultIndepIOThreads,
Msize9p: defaultMsize9p, Msize9p: defaultMsize9p,
ColdPlugVFIO: defaultColdPlugVFIO, ColdPlugVFIO: defaultColdPlugVFIO,
HotPlugVFIO: defaultHotPlugVFIO, HotPlugVFIO: defaultHotPlugVFIO,

View File

@ -840,6 +840,16 @@ func addHypervisorBlockOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig)
return err return err
} }
if err := newAnnotationConfiguration(ocispec, vcAnnotations.IndepIOThreads).setUintWithCheck(func(indepiothreads uint64) error {
if indepiothreads < 0 {
return fmt.Errorf("Error parsing annotation for indepiothreads, please specify positive numeric value")
}
sbConfig.HypervisorConfig.IndepIOThreads = uint32(indepiothreads)
return nil
}); err != nil {
return err
}
if err := newAnnotationConfiguration(ocispec, vcAnnotations.BlockDeviceCacheSet).setBool(func(blockDeviceCacheSet bool) { if err := newAnnotationConfiguration(ocispec, vcAnnotations.BlockDeviceCacheSet).setBool(func(blockDeviceCacheSet bool) {
sbConfig.HypervisorConfig.BlockDeviceCacheSet = blockDeviceCacheSet sbConfig.HypervisorConfig.BlockDeviceCacheSet = blockDeviceCacheSet
}); err != nil { }); err != nil {

View File

@ -246,9 +246,13 @@ type HypervisorConfig struct {
DisableBlockDeviceUse bool DisableBlockDeviceUse bool
// EnableIOThreads enables IO to be processed in a separate thread. // EnableIOThreads enables IO to be processed in a separate thread.
// Supported currently for virtio-scsi driver. // Supported currently for virtio-scsi and virtio-blk(depend on IndepIOThreads) driver.
EnableIOThreads bool EnableIOThreads bool
// Independent IOThreads enables IO to be processed in a separate thread, it is
// for QEMU hotplug device attach to iothread, like virtio-blk.
IndepIOThreads uint32
// Debug changes the default hypervisor and kernel parameters to // Debug changes the default hypervisor and kernel parameters to
// enable debug output where available. // enable debug output where available.
Debug bool Debug bool

View File

@ -602,9 +602,12 @@ type HypervisorConfig struct {
DisableBlockDeviceUse bool DisableBlockDeviceUse bool
// EnableIOThreads enables IO to be processed in a separate thread. // EnableIOThreads enables IO to be processed in a separate thread.
// Supported currently for virtio-scsi driver. // Supported currently for virtio-scsi driver and virtio-blk(depend on IndepIOThreads) driver.
EnableIOThreads bool EnableIOThreads bool
// Independent IOThreads enables IO to be processed in a separate thread.
IndepIOThreads uint32
// Debug changes the default hypervisor and kernel parameters to // Debug changes the default hypervisor and kernel parameters to
// enable debug output where available. // enable debug output where available.
Debug bool Debug bool

View File

@ -235,6 +235,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
BlockDeviceCacheNoflush: sconfig.HypervisorConfig.BlockDeviceCacheNoflush, BlockDeviceCacheNoflush: sconfig.HypervisorConfig.BlockDeviceCacheNoflush,
DisableBlockDeviceUse: sconfig.HypervisorConfig.DisableBlockDeviceUse, DisableBlockDeviceUse: sconfig.HypervisorConfig.DisableBlockDeviceUse,
EnableIOThreads: sconfig.HypervisorConfig.EnableIOThreads, EnableIOThreads: sconfig.HypervisorConfig.EnableIOThreads,
IndepIOThreads: sconfig.HypervisorConfig.IndepIOThreads,
Debug: sconfig.HypervisorConfig.Debug, Debug: sconfig.HypervisorConfig.Debug,
MemPrealloc: sconfig.HypervisorConfig.MemPrealloc, MemPrealloc: sconfig.HypervisorConfig.MemPrealloc,
HugePages: sconfig.HypervisorConfig.HugePages, HugePages: sconfig.HypervisorConfig.HugePages,
@ -473,6 +474,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
BlockDeviceCacheNoflush: hconf.BlockDeviceCacheNoflush, BlockDeviceCacheNoflush: hconf.BlockDeviceCacheNoflush,
DisableBlockDeviceUse: hconf.DisableBlockDeviceUse, DisableBlockDeviceUse: hconf.DisableBlockDeviceUse,
EnableIOThreads: hconf.EnableIOThreads, EnableIOThreads: hconf.EnableIOThreads,
IndepIOThreads: hconf.IndepIOThreads,
Debug: hconf.Debug, Debug: hconf.Debug,
MemPrealloc: hconf.MemPrealloc, MemPrealloc: hconf.MemPrealloc,
HugePages: hconf.HugePages, HugePages: hconf.HugePages,

View File

@ -161,9 +161,13 @@ type HypervisorConfig struct {
DisableBlockDeviceUse bool DisableBlockDeviceUse bool
// EnableIOThreads enables IO to be processed in a separate thread. // EnableIOThreads enables IO to be processed in a separate thread.
// Supported currently for virtio-scsi driver. // Supported currently for virtio-scsi and virtio-blk(depend on IndepIOThreads) driver.
EnableIOThreads bool EnableIOThreads bool
// Independent IOThreads enables IO to be processed in a separate thread, it is
// for QEMU hotplug device attach to iothread, like virtio-blk.
IndepIOThreads uint32
// Debug changes the default hypervisor and kernel parameters to // Debug changes the default hypervisor and kernel parameters to
// enable debug output where available. // enable debug output where available.
Debug bool Debug bool

View File

@ -218,9 +218,14 @@ const (
DisableBlockDeviceUse = kataAnnotHypervisorPrefix + "disable_block_device_use" DisableBlockDeviceUse = kataAnnotHypervisorPrefix + "disable_block_device_use"
// EnableIOThreads is a sandbox annotation to enable IO to be processed in a separate thread. // EnableIOThreads is a sandbox annotation to enable IO to be processed in a separate thread.
// Supported currently for virtio-scsi driver. // Supported currently for virtio-scsi and virtio-blk(depend on IndepIOThreads) driver.
EnableIOThreads = kataAnnotHypervisorPrefix + "enable_iothreads" EnableIOThreads = kataAnnotHypervisorPrefix + "enable_iothreads"
// Independent IOThreads enables IO to be processed in a separate thread, it is
// for QEMU hotplug device attach to iothread, like virtio-blk.
IndepIOThreads = kataAnnotHypervisorPrefix + "indep_iothreads"
// BlockDeviceCacheSet is a sandbox annotation that specifies cache-related options will be set to block devices or not. // BlockDeviceCacheSet is a sandbox annotation that specifies cache-related options will be set to block devices or not.
BlockDeviceCacheSet = kataAnnotHypervisorPrefix + "block_device_cache_set" BlockDeviceCacheSet = kataAnnotHypervisorPrefix + "block_device_cache_set"

View File

@ -143,6 +143,8 @@ const (
qemuStopSandboxTimeoutSecs = 15 qemuStopSandboxTimeoutSecs = 15
qomPathPrefix = "/machine/peripheral/" qomPathPrefix = "/machine/peripheral/"
indepIOThreadsPrefix = "indep_iothread"
) )
// agnostic list of kernel parameters // agnostic list of kernel parameters
@ -513,6 +515,26 @@ func (q *qemu) setupFileBackedMem(knobs *govmmQemu.Knobs, memory *govmmQemu.Memo
memory.Path = target memory.Path = target
} }
func (q *qemu) setupIoThread(ioThread *govmmQemu.IOThread) []govmmQemu.IOThread {
var tmp_threads []govmmQemu.IOThread
// Add virtio-scsi IOThreads for QEMU
if ioThread != nil {
tmp_threads = append(tmp_threads, *ioThread)
}
// Add Independent IOThreads for QEMU
if q.config.IndepIOThreads > 0 {
for i := uint32(0); i < q.config.IndepIOThreads; i++ {
id := fmt.Sprintf("%s_%d", indepIOThreadsPrefix, i)
tmp_threads = append(tmp_threads, govmmQemu.IOThread{ID: id})
}
}
return tmp_threads
}
func (q *qemu) setConfig(config *HypervisorConfig) error { func (q *qemu) setConfig(config *HypervisorConfig) error {
q.config = *config q.config = *config
@ -734,6 +756,7 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
return err return err
} }
// Note: Only virtio-SCSI device driver use this ioThread args.
devices, ioThread, kernel, err := q.buildDevices(ctx, kernelPath) devices, ioThread, kernel, err := q.buildDevices(ctx, kernelPath)
if err != nil { if err != nil {
return err return err
@ -802,9 +825,9 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
return err return err
} }
if ioThread != nil { // Setup iothread for devices.
qemuConfig.IOThreads = []govmmQemu.IOThread{*ioThread} qemuConfig.IOThreads = q.setupIoThread(ioThread)
}
// Add RNG device to hypervisor // Add RNG device to hypervisor
// Skip for s390x (as CPACF is used) or when Confidential Guest is enabled // Skip for s390x (as CPACF is used) or when Confidential Guest is enabled
if machine.Type != QemuCCWVirtio && !q.config.ConfidentialGuest { if machine.Type != QemuCCWVirtio && !q.config.ConfidentialGuest {
@ -1721,7 +1744,13 @@ func (q *qemu) hotplugAddBlockDevice(ctx context.Context, drive *config.BlockDri
queues := int(q.config.NumVCPUs()) queues := int(q.config.NumVCPUs())
if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, queues, true, defaultDisableModern); err != nil { // Make Independent IOThread 0 as the virtio-blk default.
var iothreadID string
if q.config.EnableIOThreads && q.config.IndepIOThreads > 0 {
iothreadID = fmt.Sprintf("%s_%d", indepIOThreadsPrefix, 0)
}
if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, queues, true, defaultDisableModern, iothreadID); err != nil {
return err return err
} }
case q.config.BlockDeviceDriver == config.VirtioBlockCCW: case q.config.BlockDeviceDriver == config.VirtioBlockCCW:

View File

@ -900,7 +900,7 @@ func (s *stratovirt) hotplugBlk(ctx context.Context, drive *config.BlockDrive, o
} }
devAddr := fmt.Sprintf("%d", slot) devAddr := fmt.Sprintf("%d", slot)
if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, devAddr, "", "", 0, false, false); err != nil { if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, devAddr, "", "", 0, false, false, ""); err != nil {
return err return err
} }
case RemoveDevice: case RemoveDevice: