This commit is contained in:
Zhang Chen 2025-08-12 10:14:00 +00:00 committed by GitHub
commit c5221ca4c9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 173 additions and 62 deletions

View File

@ -260,6 +260,7 @@ DEFVIRTIOFSQUEUESIZE ?= 1024
# Make sure you quote args.
DEFVIRTIOFSEXTRAARGS ?= [\"--thread-pool-size=1\", \"--announce-submounts\"]
DEFENABLEIOTHREADS := false
DEFINDEPIOTHREADS := 0
DEFENABLEVHOSTUSERSTORE := false
DEFVHOSTUSERSTOREPATH := $(PKGRUNDIR)/vhost-user
DEFVALIDVHOSTUSERSTOREPATHS := [\"$(DEFVHOSTUSERSTOREPATH)\"]
@ -731,6 +732,7 @@ USER_VARS += DEFVIRTIOFSEXTRAARGS
USER_VARS += DEFENABLEANNOTATIONS
USER_VARS += DEFENABLEANNOTATIONSTEE
USER_VARS += DEFENABLEIOTHREADS
USER_VARS += DEFINDEPIOTHREADS
USER_VARS += DEFSECCOMPSANDBOXPARAM
USER_VARS += DEFENABLEVHOSTUSERSTORE
USER_VARS += DEFVHOSTUSERSTOREPATH

View File

@ -259,11 +259,15 @@ block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
#block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
# handled in a separate IO thread. This is currently implemented
# for virtio-scsi and virtio-blk.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Independent IOThreads enables IO to be processed in a separate thread, it is
# for QEMU hotplug device attach to iothread, like virtio-blk.
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density
# as all of the memory will be allocated and locked

View File

@ -268,11 +268,15 @@ block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
#block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
# handled in a separate IO thread. This is currently implemented
# for virtio-scsi and virtio-blk.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Independent IOThreads enables IO to be processed in a separate thread, it is
# for QEMU hotplug device attach to iothread, like virtio-blk.
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density
# as all of the memory will be allocated and locked

View File

@ -252,11 +252,15 @@ block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
#block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
# handled in a separate IO thread. This is currently implemented
# for virtio-scsi and virtio-blk.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Independent IOThreads enables IO to be processed in a separate thread, it is
# for QEMU hotplug device attach to iothread, like virtio-blk.
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density
# as all of the memory will be allocated and locked

View File

@ -257,11 +257,15 @@ block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
#block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
# handled in a separate IO thread. This is currently implemented
# for virtio-scsi and virtio-blk.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Independent IOThreads enables IO to be processed in a separate thread, it is
# for QEMU hotplug device attach to iothread, like virtio-blk.
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density
# as all of the memory will be allocated and locked

View File

@ -243,11 +243,15 @@ block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
#block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
# handled in a separate IO thread. This is currently implemented
# for virtio-scsi and virtio-blk.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Independent IOThreads enables IO to be processed in a separate thread, it is
# for QEMU hotplug device attach to iothread, like virtio-blk.
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density
# as all of the memory will be allocated and locked

View File

@ -268,11 +268,15 @@ block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
#block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
# handled in a separate IO thread. This is currently implemented
# for virtio-scsi and virtio-blk.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Independent IOThreads enables IO to be processed in a separate thread, it is
# for QEMU hotplug device attach to iothread, like virtio-blk.
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density
# as all of the memory will be allocated and locked

View File

@ -253,11 +253,15 @@ block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
#block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
# handled in a separate IO thread. This is currently implemented
# for virtio-scsi and virtio-blk.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Independent IOThreads enables IO to be processed in a separate thread, it is
# for QEMU hotplug device attach to iothread, like virtio-blk.
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density
# as all of the memory will be allocated and locked

View File

@ -258,11 +258,15 @@ block_device_aio = "@DEFBLOCKDEVICEAIO_QEMU@"
#block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
# handled in a separate IO thread. This is currently implemented
# for virtio-scsi and virtio-blk.
#
enable_iothreads = @DEFENABLEIOTHREADS@
# Independent IOThreads enables IO to be processed in a separate thread, it is
# for QEMU hotplug device attach to iothread, like virtio-blk.
indep_iothreads = @DEFINDEPIOTHREADS@
# Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density
# as all of the memory will be allocated and locked

View File

@ -1102,7 +1102,7 @@ func (q *QMP) ExecuteDeviceDel(ctx context.Context, devID string) error {
// disableModern indicates if virtio version 1.0 should be replaced by the
// former version 0.9, as there is a KVM bug that occurs when using virtio
// 1.0 in nested environments.
func (q *QMP) ExecutePCIDeviceAdd(ctx context.Context, blockdevID, devID, driver, addr, bus, romfile string, queues int, shared, disableModern bool) error {
func (q *QMP) ExecutePCIDeviceAdd(ctx context.Context, blockdevID, devID, driver, addr, bus, romfile string, queues int, shared, disableModern bool, iothreadID string) error {
args := map[string]interface{}{
"id": devID,
"driver": driver,
@ -1129,6 +1129,10 @@ func (q *QMP) ExecutePCIDeviceAdd(ctx context.Context, blockdevID, devID, driver
}
}
if iothreadID != "" {
args["iothread"] = iothreadID
}
return q.executeCommand(ctx, "device_add", args, nil)
}

View File

@ -1066,7 +1066,7 @@ func TestQMPPCIDeviceAdd(t *testing.T) {
blockdevID := fmt.Sprintf("drive_%s", volumeUUID)
devID := fmt.Sprintf("device_%s", volumeUUID)
err := q.ExecutePCIDeviceAdd(context.Background(), blockdevID, devID,
"virtio-blk-pci", "0x1", "", "", 1, true, false)
"virtio-blk-pci", "0x1", "", "", 1, true, false, "")
if err != nil {
t.Fatalf("Unexpected error %v", err)
}

View File

@ -207,41 +207,42 @@ const (
)
type RuntimeConfigOptions struct {
Hypervisor string
HypervisorPath string
DefaultGuestHookPath string
KernelPath string
ImagePath string
RootfsType string
KernelParams string
MachineType string
LogPath string
BlockDeviceDriver string
BlockDeviceAIO string
SharedFS string
VirtioFSDaemon string
JaegerEndpoint string
JaegerUser string
JaegerPassword string
PFlash []string
HotPlugVFIO config.PCIePort
ColdPlugVFIO config.PCIePort
PCIeRootPort uint32
PCIeSwitchPort uint32
DefaultVCPUCount uint32
DefaultMaxVCPUCount uint32
DefaultMemSize uint32
DefaultMaxMemorySize uint64
DefaultMsize9p uint32
DisableBlock bool
EnableIOThreads bool
DisableNewNetNs bool
HypervisorDebug bool
RuntimeDebug bool
RuntimeTrace bool
AgentDebug bool
AgentTrace bool
EnablePprof bool
Hypervisor string
HypervisorPath string
DefaultGuestHookPath string
KernelPath string
ImagePath string
RootfsType string
KernelParams string
MachineType string
LogPath string
BlockDeviceDriver string
BlockDeviceAIO string
SharedFS string
VirtioFSDaemon string
JaegerEndpoint string
JaegerUser string
JaegerPassword string
PFlash []string
HotPlugVFIO config.PCIePort
ColdPlugVFIO config.PCIePort
PCIeRootPort uint32
PCIeSwitchPort uint32
DefaultVCPUCount uint32
DefaultMaxVCPUCount uint32
DefaultMemSize uint32
DefaultMaxMemorySize uint64
DefaultMsize9p uint32
DefaultIndepIOThreads uint32
DisableBlock bool
EnableIOThreads bool
DisableNewNetNs bool
HypervisorDebug bool
RuntimeDebug bool
RuntimeTrace bool
AgentDebug bool
AgentTrace bool
EnablePprof bool
}
// ContainerIDTestDataType is a type used to test Container and Sandbox ID's.
@ -318,6 +319,7 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
default_memory = ` + strconv.FormatUint(uint64(config.DefaultMemSize), 10) + `
disable_block_device_use = ` + strconv.FormatBool(config.DisableBlock) + `
enable_iothreads = ` + strconv.FormatBool(config.EnableIOThreads) + `
indep_iothreads = ` + strconv.FormatUint(uint64(config.DefaultIndepIOThreads), 10) + `
cold_plug_vfio = "` + config.ColdPlugVFIO.String() + `"
hot_plug_vfio = "` + config.HotPlugVFIO.String() + `"
pcie_root_port = ` + strconv.FormatUint(uint64(config.PCIeRootPort), 10) + `

View File

@ -75,6 +75,7 @@ const defaultBlockDeviceCacheSet bool = false
const defaultBlockDeviceCacheDirect bool = false
const defaultBlockDeviceCacheNoflush bool = false
const defaultEnableIOThreads bool = false
const defaultIndepIOThreads uint32 = 0
const defaultEnableMemPrealloc bool = false
const defaultEnableReclaimGuestFreedMemory bool = false
const defaultEnableHugePages bool = false

View File

@ -155,6 +155,7 @@ type hypervisor struct {
Debug bool `toml:"enable_debug"`
DisableNestingChecks bool `toml:"disable_nesting_checks"`
EnableIOThreads bool `toml:"enable_iothreads"`
IndepIOThreads uint32 `toml:"indep_iothreads"`
DisableImageNvdimm bool `toml:"disable_image_nvdimm"`
HotPlugVFIO config.PCIePort `toml:"hot_plug_vfio"`
ColdPlugVFIO config.PCIePort `toml:"cold_plug_vfio"`
@ -614,6 +615,14 @@ func (h hypervisor) msize9p() uint32 {
return h.Msize9p
}
func (h hypervisor) indepiothreads() uint32 {
if h.IndepIOThreads == 0 {
return defaultIndepIOThreads
}
return h.IndepIOThreads
}
func (h hypervisor) guestHookPath() string {
if h.GuestHookPath == "" {
return defaultGuestHookPath
@ -810,6 +819,7 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
DisableNestingChecks: h.DisableNestingChecks,
BlockDeviceDriver: blockDriver,
EnableIOThreads: h.EnableIOThreads,
IndepIOThreads: h.indepiothreads(),
DisableVhostNet: true, // vhost-net backend is not supported in Firecracker
GuestHookPath: h.guestHookPath(),
RxRateLimiterMaxRate: rxRateLimiterMaxRate,
@ -964,6 +974,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
BlockDeviceCacheDirect: h.BlockDeviceCacheDirect,
BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush,
EnableIOThreads: h.EnableIOThreads,
IndepIOThreads: h.indepiothreads(),
Msize9p: h.msize9p(),
DisableImageNvdimm: h.DisableImageNvdimm,
HotPlugVFIO: h.hotPlugVFIO(),
@ -1094,6 +1105,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
BlockDeviceCacheSet: h.BlockDeviceCacheSet,
BlockDeviceCacheDirect: h.BlockDeviceCacheDirect,
EnableIOThreads: h.EnableIOThreads,
IndepIOThreads: h.indepiothreads(),
Msize9p: h.msize9p(),
DisableImageNvdimm: h.DisableImageNvdimm,
ColdPlugVFIO: h.coldPlugVFIO(),
@ -1452,6 +1464,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
BlockDeviceCacheDirect: defaultBlockDeviceCacheDirect,
BlockDeviceCacheNoflush: defaultBlockDeviceCacheNoflush,
EnableIOThreads: defaultEnableIOThreads,
IndepIOThreads: defaultIndepIOThreads,
Msize9p: defaultMsize9p,
ColdPlugVFIO: defaultColdPlugVFIO,
HotPlugVFIO: defaultHotPlugVFIO,

View File

@ -840,6 +840,16 @@ func addHypervisorBlockOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig)
return err
}
if err := newAnnotationConfiguration(ocispec, vcAnnotations.IndepIOThreads).setUintWithCheck(func(indepiothreads uint64) error {
if indepiothreads < 0 {
return fmt.Errorf("Error parsing annotation for indepiothreads, please specify positive numeric value")
}
sbConfig.HypervisorConfig.IndepIOThreads = uint32(indepiothreads)
return nil
}); err != nil {
return err
}
if err := newAnnotationConfiguration(ocispec, vcAnnotations.BlockDeviceCacheSet).setBool(func(blockDeviceCacheSet bool) {
sbConfig.HypervisorConfig.BlockDeviceCacheSet = blockDeviceCacheSet
}); err != nil {

View File

@ -246,9 +246,13 @@ type HypervisorConfig struct {
DisableBlockDeviceUse bool
// EnableIOThreads enables IO to be processed in a separate thread.
// Supported currently for virtio-scsi driver.
// Supported currently for virtio-scsi and virtio-blk(depend on IndepIOThreads) driver.
EnableIOThreads bool
// Independent IOThreads enables IO to be processed in a separate thread, it is
// for QEMU hotplug device attach to iothread, like virtio-blk.
IndepIOThreads uint32
// Debug changes the default hypervisor and kernel parameters to
// enable debug output where available.
Debug bool

View File

@ -602,9 +602,12 @@ type HypervisorConfig struct {
DisableBlockDeviceUse bool
// EnableIOThreads enables IO to be processed in a separate thread.
// Supported currently for virtio-scsi driver.
// Supported currently for virtio-scsi driver and virtio-blk(depend on IndepIOThreads) driver.
EnableIOThreads bool
// Independent IOThreads enables IO to be processed in a separate thread.
IndepIOThreads uint32
// Debug changes the default hypervisor and kernel parameters to
// enable debug output where available.
Debug bool

View File

@ -235,6 +235,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
BlockDeviceCacheNoflush: sconfig.HypervisorConfig.BlockDeviceCacheNoflush,
DisableBlockDeviceUse: sconfig.HypervisorConfig.DisableBlockDeviceUse,
EnableIOThreads: sconfig.HypervisorConfig.EnableIOThreads,
IndepIOThreads: sconfig.HypervisorConfig.IndepIOThreads,
Debug: sconfig.HypervisorConfig.Debug,
MemPrealloc: sconfig.HypervisorConfig.MemPrealloc,
HugePages: sconfig.HypervisorConfig.HugePages,
@ -473,6 +474,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
BlockDeviceCacheNoflush: hconf.BlockDeviceCacheNoflush,
DisableBlockDeviceUse: hconf.DisableBlockDeviceUse,
EnableIOThreads: hconf.EnableIOThreads,
IndepIOThreads: hconf.IndepIOThreads,
Debug: hconf.Debug,
MemPrealloc: hconf.MemPrealloc,
HugePages: hconf.HugePages,

View File

@ -161,9 +161,13 @@ type HypervisorConfig struct {
DisableBlockDeviceUse bool
// EnableIOThreads enables IO to be processed in a separate thread.
// Supported currently for virtio-scsi driver.
// Supported currently for virtio-scsi and virtio-blk(depend on IndepIOThreads) driver.
EnableIOThreads bool
// Independent IOThreads enables IO to be processed in a separate thread, it is
// for QEMU hotplug device attach to iothread, like virtio-blk.
IndepIOThreads uint32
// Debug changes the default hypervisor and kernel parameters to
// enable debug output where available.
Debug bool

View File

@ -218,9 +218,14 @@ const (
DisableBlockDeviceUse = kataAnnotHypervisorPrefix + "disable_block_device_use"
// EnableIOThreads is a sandbox annotation to enable IO to be processed in a separate thread.
// Supported currently for virtio-scsi driver.
// Supported currently for virtio-scsi and virtio-blk(depend on IndepIOThreads) driver.
EnableIOThreads = kataAnnotHypervisorPrefix + "enable_iothreads"
// Independent IOThreads enables IO to be processed in a separate thread, it is
// for QEMU hotplug device attach to iothread, like virtio-blk.
IndepIOThreads = kataAnnotHypervisorPrefix + "indep_iothreads"
// BlockDeviceCacheSet is a sandbox annotation that specifies cache-related options will be set to block devices or not.
BlockDeviceCacheSet = kataAnnotHypervisorPrefix + "block_device_cache_set"

View File

@ -143,6 +143,8 @@ const (
qemuStopSandboxTimeoutSecs = 15
qomPathPrefix = "/machine/peripheral/"
indepIOThreadsPrefix = "indep_iothread"
)
// agnostic list of kernel parameters
@ -513,6 +515,26 @@ func (q *qemu) setupFileBackedMem(knobs *govmmQemu.Knobs, memory *govmmQemu.Memo
memory.Path = target
}
func (q *qemu) setupIoThread(ioThread *govmmQemu.IOThread) []govmmQemu.IOThread {
var tmp_threads []govmmQemu.IOThread
// Add virtio-scsi IOThreads for QEMU
if ioThread != nil {
tmp_threads = append(tmp_threads, *ioThread)
}
// Add Independent IOThreads for QEMU
if q.config.IndepIOThreads > 0 {
for i := uint32(0); i < q.config.IndepIOThreads; i++ {
id := fmt.Sprintf("%s_%d", indepIOThreadsPrefix, i)
tmp_threads = append(tmp_threads, govmmQemu.IOThread{ID: id})
}
}
return tmp_threads
}
func (q *qemu) setConfig(config *HypervisorConfig) error {
q.config = *config
@ -734,6 +756,7 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
return err
}
// Note: Only virtio-SCSI device driver use this ioThread args.
devices, ioThread, kernel, err := q.buildDevices(ctx, kernelPath)
if err != nil {
return err
@ -802,9 +825,9 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
return err
}
if ioThread != nil {
qemuConfig.IOThreads = []govmmQemu.IOThread{*ioThread}
}
// Setup iothread for devices.
qemuConfig.IOThreads = q.setupIoThread(ioThread)
// Add RNG device to hypervisor
// Skip for s390x (as CPACF is used) or when Confidential Guest is enabled
if machine.Type != QemuCCWVirtio && !q.config.ConfidentialGuest {
@ -1721,7 +1744,13 @@ func (q *qemu) hotplugAddBlockDevice(ctx context.Context, drive *config.BlockDri
queues := int(q.config.NumVCPUs())
if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, queues, true, defaultDisableModern); err != nil {
// Make Independent IOThread 0 as the virtio-blk default.
var iothreadID string
if q.config.EnableIOThreads && q.config.IndepIOThreads > 0 {
iothreadID = fmt.Sprintf("%s_%d", indepIOThreadsPrefix, 0)
}
if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, queues, true, defaultDisableModern, iothreadID); err != nil {
return err
}
case q.config.BlockDeviceDriver == config.VirtioBlockCCW:

View File

@ -900,7 +900,7 @@ func (s *stratovirt) hotplugBlk(ctx context.Context, drive *config.BlockDrive, o
}
devAddr := fmt.Sprintf("%d", slot)
if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, devAddr, "", "", 0, false, false); err != nil {
if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, devAddr, "", "", 0, false, false, ""); err != nil {
return err
}
case RemoveDevice: