diff --git a/src/runtime/Makefile b/src/runtime/Makefile index ef071c15b7..a96ee49dc6 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -193,6 +193,7 @@ DEFVALIDVIRTIOFSDAEMONPATHS := [\"$(DEFVIRTIOFSDAEMON)\"] #if value is 0, DAX is not enabled DEFVIRTIOFSCACHESIZE ?= 0 DEFVIRTIOFSCACHE ?= auto +DEFVIRTIOFSQUEUESIZE ?= 1024 # Format example: # [\"-o\", \"arg1=xxx,arg2\", \"-o\", \"hello world\", \"--arg3=yyy\"] # @@ -471,6 +472,7 @@ USER_VARS += DEFVIRTIOFSDAEMON USER_VARS += DEFVALIDVIRTIOFSDAEMONPATHS USER_VARS += DEFVIRTIOFSCACHESIZE USER_VARS += DEFVIRTIOFSCACHE +USER_VARS += DEFVIRTIOFSQUEUESIZE USER_VARS += DEFVIRTIOFSEXTRAARGS USER_VARS += DEFENABLEANNOTATIONS USER_VARS += DEFENABLEIOTHREADS diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index 59ddf43e12..e47a1d92a0 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -126,6 +126,9 @@ valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@ # Default size of DAX cache in MiB virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ +# Default size of virtqueues +virtio_fs_queue_size = @DEFVIRTIOFSQUEUESIZE@ + # Extra args for virtiofsd daemon # # Format example: diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index 7d0487d2bd..8330042977 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -190,6 +190,9 @@ valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@ # Default size of DAX cache in MiB virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@ +# Default size of virtqueues +virtio_fs_queue_size = @DEFVIRTIOFSQUEUESIZE@ + # Extra args for virtiofsd daemon # # Format example: diff --git a/src/runtime/pkg/device/config/config.go b/src/runtime/pkg/device/config/config.go index a11c52f75c..f1078671ae 100644 --- a/src/runtime/pkg/device/config/config.go +++ b/src/runtime/pkg/device/config/config.go @@ -318,6 +318,8 @@ type VhostUserDeviceAttrs struct { Index int CacheSize uint32 + + QueueSize uint32 } // GetHostPathFunc is function pointer used to mock GetHostPath in tests. diff --git a/src/runtime/pkg/govmm/qemu/qemu.go b/src/runtime/pkg/govmm/qemu/qemu.go index 5b04a01c1d..420b5c45e8 100644 --- a/src/runtime/pkg/govmm/qemu/qemu.go +++ b/src/runtime/pkg/govmm/qemu/qemu.go @@ -1336,6 +1336,7 @@ type VhostUserDevice struct { Address string //used for MAC address in net case Tag string //virtio-fs volume id for mounting inside guest CacheSize uint32 //virtio-fs DAX cache size in MiB + QueueSize uint32 //size of virtqueues SharedVersions bool //enable virtio-fs shared version metadata VhostUserType DeviceDriver @@ -1503,6 +1504,11 @@ func (vhostuserDev VhostUserDevice) QemuFSParams(config *Config) []string { deviceParams = append(deviceParams, driver) deviceParams = append(deviceParams, fmt.Sprintf("chardev=%s", vhostuserDev.CharDevID)) deviceParams = append(deviceParams, fmt.Sprintf("tag=%s", vhostuserDev.Tag)) + queueSize := uint32(1024) + if vhostuserDev.QueueSize != 0 { + queueSize = vhostuserDev.QueueSize + } + deviceParams = append(deviceParams, fmt.Sprintf("queue-size=%d", queueSize)) if vhostuserDev.CacheSize != 0 { deviceParams = append(deviceParams, fmt.Sprintf("cache-size=%dM", vhostuserDev.CacheSize)) } diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index d334c61713..5d923f8fd8 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -114,6 +114,7 @@ type hypervisor struct { RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"` TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"` MemOffset uint64 `toml:"memory_offset"` + DefaultMaxMemorySize uint64 `toml:"default_maxmemory"` DiskRateLimiterBwMaxRate int64 `toml:"disk_rate_limiter_bw_max_rate"` DiskRateLimiterBwOneTimeBurst int64 `toml:"disk_rate_limiter_bw_one_time_burst"` DiskRateLimiterOpsMaxRate int64 `toml:"disk_rate_limiter_ops_max_rate"` @@ -123,10 +124,10 @@ type hypervisor struct { NetRateLimiterOpsMaxRate int64 `toml:"net_rate_limiter_ops_max_rate"` NetRateLimiterOpsOneTimeBurst int64 `toml:"net_rate_limiter_ops_one_time_burst"` VirtioFSCacheSize uint32 `toml:"virtio_fs_cache_size"` + VirtioFSQueueSize uint32 `toml:"virtio_fs_queue_size"` DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` MemorySize uint32 `toml:"default_memory"` MemSlots uint32 `toml:"memory_slots"` - DefaultMaxMemorySize uint64 `toml:"default_maxmemory"` DefaultBridges uint32 `toml:"default_bridges"` Msize9p uint32 `toml:"msize_9p"` PCIeRootPort uint32 `toml:"pcie_root_port"` @@ -798,6 +799,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { VirtioFSDaemonList: h.VirtioFSDaemonList, VirtioFSCacheSize: h.VirtioFSCacheSize, VirtioFSCache: h.defaultVirtioFSCache(), + VirtioFSQueueSize: h.VirtioFSQueueSize, VirtioFSExtraArgs: h.VirtioFSExtraArgs, MemPrealloc: h.MemPrealloc, HugePages: h.HugePages, diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index a0f08e8cb7..b9aa31e165 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -1587,6 +1587,9 @@ func (clh *cloudHypervisor) addVolume(volume types.Volume) error { // default values defined by cloud-hypervisor numQueues := int32(1) queueSize := int32(1024) + if clh.config.VirtioFSQueueSize != 0 { + queueSize = int32(clh.config.VirtioFSQueueSize) + } fs := chclient.NewFsConfig(volume.MountTag, vfsdSockPath, numQueues, queueSize) clh.vmconfig.Fs = &[]chclient.FsConfig{*fs} diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index eb5abbb3b3..548ce6f77d 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -467,6 +467,9 @@ type HypervisorConfig struct { // VirtioFSCacheSize is the DAX cache size in MiB VirtioFSCacheSize uint32 + // Size of virtqueues + VirtioFSQueueSize uint32 + // User ID. Uid uint32 diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index b7f9601c13..deec2f09c0 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -2080,6 +2080,7 @@ func (q *qemu) AddDevice(ctx context.Context, devInfo interface{}, devType Devic Type: config.VhostUserFS, CacheSize: q.config.VirtioFSCacheSize, Cache: q.config.VirtioFSCache, + QueueSize: q.config.VirtioFSQueueSize, } vhostDev.SocketPath = sockPath vhostDev.DevID = id diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index 1d7d76bfae..07282afa78 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -663,6 +663,7 @@ func (q *qemuArchBase) appendVhostUserDevice(ctx context.Context, devices []govm qemuVhostUserDevice.TypeDevID = utils.MakeNameID("fs", attr.DevID, maxDevIDSize) qemuVhostUserDevice.Tag = attr.Tag qemuVhostUserDevice.CacheSize = attr.CacheSize + qemuVhostUserDevice.QueueSize = attr.QueueSize qemuVhostUserDevice.VhostUserType = govmmQemu.VhostUserFS }