diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 940338c8e6..192093c74b 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -260,6 +260,7 @@ DEFVIRTIOFSQUEUESIZE ?= 1024 # Make sure you quote args. DEFVIRTIOFSEXTRAARGS ?= [\"--thread-pool-size=1\", \"--announce-submounts\"] DEFENABLEIOTHREADS := false +DEFINDEPIOTHREADS := 0 DEFENABLEVHOSTUSERSTORE := false DEFVHOSTUSERSTOREPATH := $(PKGRUNDIR)/vhost-user DEFVALIDVHOSTUSERSTOREPATHS := [\"$(DEFVHOSTUSERSTOREPATH)\"] @@ -731,6 +732,7 @@ USER_VARS += DEFVIRTIOFSEXTRAARGS USER_VARS += DEFENABLEANNOTATIONS USER_VARS += DEFENABLEANNOTATIONSTEE USER_VARS += DEFENABLEIOTHREADS +USER_VARS += DEFINDEPIOTHREADS USER_VARS += DEFSECCOMPSANDBOXPARAM USER_VARS += DEFENABLEVHOSTUSERSTORE USER_VARS += DEFVHOSTUSERSTOREPATH diff --git a/src/runtime/pkg/katatestutils/utils.go b/src/runtime/pkg/katatestutils/utils.go index c855ae9fbe..fd189dfd33 100644 --- a/src/runtime/pkg/katatestutils/utils.go +++ b/src/runtime/pkg/katatestutils/utils.go @@ -207,41 +207,42 @@ const ( ) type RuntimeConfigOptions struct { - Hypervisor string - HypervisorPath string - DefaultGuestHookPath string - KernelPath string - ImagePath string - RootfsType string - KernelParams string - MachineType string - LogPath string - BlockDeviceDriver string - BlockDeviceAIO string - SharedFS string - VirtioFSDaemon string - JaegerEndpoint string - JaegerUser string - JaegerPassword string - PFlash []string - HotPlugVFIO config.PCIePort - ColdPlugVFIO config.PCIePort - PCIeRootPort uint32 - PCIeSwitchPort uint32 - DefaultVCPUCount uint32 - DefaultMaxVCPUCount uint32 - DefaultMemSize uint32 - DefaultMaxMemorySize uint64 - DefaultMsize9p uint32 - DisableBlock bool - EnableIOThreads bool - DisableNewNetNs bool - HypervisorDebug bool - RuntimeDebug bool - RuntimeTrace bool - AgentDebug bool - AgentTrace bool - EnablePprof bool + Hypervisor string + HypervisorPath string + DefaultGuestHookPath string + KernelPath string + ImagePath string + RootfsType string + KernelParams string + MachineType string + LogPath string + BlockDeviceDriver string + BlockDeviceAIO string + SharedFS string + VirtioFSDaemon string + JaegerEndpoint string + JaegerUser string + JaegerPassword string + PFlash []string + HotPlugVFIO config.PCIePort + ColdPlugVFIO config.PCIePort + PCIeRootPort uint32 + PCIeSwitchPort uint32 + DefaultVCPUCount uint32 + DefaultMaxVCPUCount uint32 + DefaultMemSize uint32 + DefaultMaxMemorySize uint64 + DefaultMsize9p uint32 + DefaultIndepIOThreads uint32 + DisableBlock bool + EnableIOThreads bool + DisableNewNetNs bool + HypervisorDebug bool + RuntimeDebug bool + RuntimeTrace bool + AgentDebug bool + AgentTrace bool + EnablePprof bool } // ContainerIDTestDataType is a type used to test Container and Sandbox ID's. @@ -318,6 +319,7 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string { default_memory = ` + strconv.FormatUint(uint64(config.DefaultMemSize), 10) + ` disable_block_device_use = ` + strconv.FormatBool(config.DisableBlock) + ` enable_iothreads = ` + strconv.FormatBool(config.EnableIOThreads) + ` + indep_iothreads = ` + strconv.FormatUint(uint64(config.DefaultIndepIOThreads), 10) + ` cold_plug_vfio = "` + config.ColdPlugVFIO.String() + `" hot_plug_vfio = "` + config.HotPlugVFIO.String() + `" pcie_root_port = ` + strconv.FormatUint(uint64(config.PCIeRootPort), 10) + ` diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in index d7680dd1bd..cecce93f18 100644 --- a/src/runtime/pkg/katautils/config-settings.go.in +++ b/src/runtime/pkg/katautils/config-settings.go.in @@ -75,6 +75,7 @@ const defaultBlockDeviceCacheSet bool = false const defaultBlockDeviceCacheDirect bool = false const defaultBlockDeviceCacheNoflush bool = false const defaultEnableIOThreads bool = false +const defaultIndepIOThreads uint32 = 0 const defaultEnableMemPrealloc bool = false const defaultEnableReclaimGuestFreedMemory bool = false const defaultEnableHugePages bool = false diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index f15d945ca9..35a592680b 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -155,6 +155,7 @@ type hypervisor struct { Debug bool `toml:"enable_debug"` DisableNestingChecks bool `toml:"disable_nesting_checks"` EnableIOThreads bool `toml:"enable_iothreads"` + IndepIOThreads uint32 `toml:"indep_iothreads"` DisableImageNvdimm bool `toml:"disable_image_nvdimm"` HotPlugVFIO config.PCIePort `toml:"hot_plug_vfio"` ColdPlugVFIO config.PCIePort `toml:"cold_plug_vfio"` @@ -614,6 +615,14 @@ func (h hypervisor) msize9p() uint32 { return h.Msize9p } +func (h hypervisor) indepiothreads() uint32 { + if h.IndepIOThreads == 0 { + return defaultIndepIOThreads + } + + return h.IndepIOThreads +} + func (h hypervisor) guestHookPath() string { if h.GuestHookPath == "" { return defaultGuestHookPath @@ -810,6 +819,7 @@ func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { DisableNestingChecks: h.DisableNestingChecks, BlockDeviceDriver: blockDriver, EnableIOThreads: h.EnableIOThreads, + IndepIOThreads: h.indepiothreads(), DisableVhostNet: true, // vhost-net backend is not supported in Firecracker GuestHookPath: h.guestHookPath(), RxRateLimiterMaxRate: rxRateLimiterMaxRate, @@ -964,6 +974,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { BlockDeviceCacheDirect: h.BlockDeviceCacheDirect, BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush, EnableIOThreads: h.EnableIOThreads, + IndepIOThreads: h.indepiothreads(), Msize9p: h.msize9p(), DisableImageNvdimm: h.DisableImageNvdimm, HotPlugVFIO: h.hotPlugVFIO(), @@ -1094,6 +1105,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { BlockDeviceCacheSet: h.BlockDeviceCacheSet, BlockDeviceCacheDirect: h.BlockDeviceCacheDirect, EnableIOThreads: h.EnableIOThreads, + IndepIOThreads: h.indepiothreads(), Msize9p: h.msize9p(), DisableImageNvdimm: h.DisableImageNvdimm, ColdPlugVFIO: h.coldPlugVFIO(), @@ -1452,6 +1464,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig { BlockDeviceCacheDirect: defaultBlockDeviceCacheDirect, BlockDeviceCacheNoflush: defaultBlockDeviceCacheNoflush, EnableIOThreads: defaultEnableIOThreads, + IndepIOThreads: defaultIndepIOThreads, Msize9p: defaultMsize9p, ColdPlugVFIO: defaultColdPlugVFIO, HotPlugVFIO: defaultHotPlugVFIO, diff --git a/src/runtime/pkg/oci/utils.go b/src/runtime/pkg/oci/utils.go index d49aabd988..ee5272a53c 100644 --- a/src/runtime/pkg/oci/utils.go +++ b/src/runtime/pkg/oci/utils.go @@ -840,6 +840,16 @@ func addHypervisorBlockOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) return err } + if err := newAnnotationConfiguration(ocispec, vcAnnotations.IndepIOThreads).setUintWithCheck(func(indepiothreads uint64) error { + if indepiothreads < 0 { + return fmt.Errorf("Error parsing annotation for indepiothreads, please specify positive numeric value") + } + sbConfig.HypervisorConfig.IndepIOThreads = uint32(indepiothreads) + return nil + }); err != nil { + return err + } + if err := newAnnotationConfiguration(ocispec, vcAnnotations.BlockDeviceCacheSet).setBool(func(blockDeviceCacheSet bool) { sbConfig.HypervisorConfig.BlockDeviceCacheSet = blockDeviceCacheSet }); err != nil { diff --git a/src/runtime/virtcontainers/documentation/api/1.0/api.md b/src/runtime/virtcontainers/documentation/api/1.0/api.md index ad336d359b..953c837af7 100644 --- a/src/runtime/virtcontainers/documentation/api/1.0/api.md +++ b/src/runtime/virtcontainers/documentation/api/1.0/api.md @@ -249,6 +249,10 @@ type HypervisorConfig struct { // Supported currently for virtio-scsi driver. EnableIOThreads bool + // Independent IOThreads enables IO to be processed in a separate thread, it is + // for QEMU hotplug device attach to iothread, like virtio-blk. + IndepIOThreads uint32 + // Debug changes the default hypervisor and kernel parameters to // enable debug output where available. Debug bool diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 22423ab122..e7e294e4c8 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -605,6 +605,9 @@ type HypervisorConfig struct { // Supported currently for virtio-scsi driver. EnableIOThreads bool + // Independent IOThreads enables IO to be processed in a separate thread. + IndepIOThreads uint32 + // Debug changes the default hypervisor and kernel parameters to // enable debug output where available. Debug bool diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index 79731b7998..a3cb0bc859 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -235,6 +235,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { BlockDeviceCacheNoflush: sconfig.HypervisorConfig.BlockDeviceCacheNoflush, DisableBlockDeviceUse: sconfig.HypervisorConfig.DisableBlockDeviceUse, EnableIOThreads: sconfig.HypervisorConfig.EnableIOThreads, + IndepIOThreads: sconfig.HypervisorConfig.IndepIOThreads, Debug: sconfig.HypervisorConfig.Debug, MemPrealloc: sconfig.HypervisorConfig.MemPrealloc, HugePages: sconfig.HypervisorConfig.HugePages, @@ -473,6 +474,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { BlockDeviceCacheNoflush: hconf.BlockDeviceCacheNoflush, DisableBlockDeviceUse: hconf.DisableBlockDeviceUse, EnableIOThreads: hconf.EnableIOThreads, + IndepIOThreads: hconf.IndepIOThreads, Debug: hconf.Debug, MemPrealloc: hconf.MemPrealloc, HugePages: hconf.HugePages, diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go index 194456554f..e9284d30d8 100644 --- a/src/runtime/virtcontainers/persist/api/config.go +++ b/src/runtime/virtcontainers/persist/api/config.go @@ -164,6 +164,10 @@ type HypervisorConfig struct { // Supported currently for virtio-scsi driver. EnableIOThreads bool + // Independent IOThreads enables IO to be processed in a separate thread, it is + // for QEMU hotplug device attach to iothread, like virtio-blk. + IndepIOThreads uint32 + // Debug changes the default hypervisor and kernel parameters to // enable debug output where available. Debug bool diff --git a/src/runtime/virtcontainers/pkg/annotations/annotations.go b/src/runtime/virtcontainers/pkg/annotations/annotations.go index 03b9e9b70c..d32a14ed1a 100644 --- a/src/runtime/virtcontainers/pkg/annotations/annotations.go +++ b/src/runtime/virtcontainers/pkg/annotations/annotations.go @@ -221,6 +221,11 @@ const ( // Supported currently for virtio-scsi driver. EnableIOThreads = kataAnnotHypervisorPrefix + "enable_iothreads" + // Independent IOThreads enables IO to be processed in a separate thread, it is + // for QEMU hotplug device attach to iothread, like virtio-blk. + IndepIOThreads = kataAnnotHypervisorPrefix + "indep_iothreads" + + // BlockDeviceCacheSet is a sandbox annotation that specifies cache-related options will be set to block devices or not. BlockDeviceCacheSet = kataAnnotHypervisorPrefix + "block_device_cache_set"