runtime: Support virtiofs queue size for qemu and make it configurable

The default vhost-user-fs queue-size of qemu is 128 now. Set it to 1024
by default which is same as clh. Also make this value configurable.

Fixes: #5694

Signed-off-by: liyuxuan.darfux <liyuxuan.darfux@bytedance.com>

src/runtime/Makefile

@@ -193,6 +193,7 @@ DEFVALIDVIRTIOFSDAEMONPATHS := [\"$(DEFVIRTIOFSDAEMON)\"]
 #if value is 0, DAX is not enabled
 DEFVIRTIOFSCACHESIZE ?= 0
 DEFVIRTIOFSCACHE ?= auto
+DEFVIRTIOFSQUEUESIZE ?= 1024
 # Format example:
 #   [\"-o\", \"arg1=xxx,arg2\", \"-o\", \"hello world\", \"--arg3=yyy\"]
 #
@@ -471,6 +472,7 @@ USER_VARS += DEFVIRTIOFSDAEMON
 USER_VARS += DEFVALIDVIRTIOFSDAEMONPATHS
 USER_VARS += DEFVIRTIOFSCACHESIZE
 USER_VARS += DEFVIRTIOFSCACHE
+USER_VARS += DEFVIRTIOFSQUEUESIZE
 USER_VARS += DEFVIRTIOFSEXTRAARGS
 USER_VARS += DEFENABLEANNOTATIONS
 USER_VARS += DEFENABLEIOTHREADS

src/runtime/config/configuration-clh.toml.in

@@ -126,6 +126,9 @@ valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@
 # Default size of DAX cache in MiB
 virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
 
+# Default size of virtqueues
+virtio_fs_queue_size = @DEFVIRTIOFSQUEUESIZE@
+
 # Extra args for virtiofsd daemon
 #
 # Format example:

src/runtime/config/configuration-qemu.toml.in

@@ -190,6 +190,9 @@ valid_virtio_fs_daemon_paths = @DEFVALIDVIRTIOFSDAEMONPATHS@
 # Default size of DAX cache in MiB
 virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
 
+# Default size of virtqueues
+virtio_fs_queue_size = @DEFVIRTIOFSQUEUESIZE@
+
 # Extra args for virtiofsd daemon
 #
 # Format example:

src/runtime/pkg/device/config/config.go

@@ -318,6 +318,8 @@ type VhostUserDeviceAttrs struct {
 	Index int
 
 	CacheSize uint32
+
+	QueueSize uint32
 }
 
 // GetHostPathFunc is function pointer used to mock GetHostPath in tests.

src/runtime/pkg/govmm/qemu/qemu.go

@@ -1336,6 +1336,7 @@ type VhostUserDevice struct {
 	Address        string //used for MAC address in net case
 	Tag            string //virtio-fs volume id for mounting inside guest
 	CacheSize      uint32 //virtio-fs DAX cache size in MiB
+	QueueSize      uint32 //size of virtqueues
 	SharedVersions bool   //enable virtio-fs shared version metadata
 	VhostUserType  DeviceDriver
 
@@ -1503,6 +1504,11 @@ func (vhostuserDev VhostUserDevice) QemuFSParams(config *Config) []string {
 	deviceParams = append(deviceParams, driver)
 	deviceParams = append(deviceParams, fmt.Sprintf("chardev=%s", vhostuserDev.CharDevID))
 	deviceParams = append(deviceParams, fmt.Sprintf("tag=%s", vhostuserDev.Tag))
+	queueSize := uint32(1024)
+	if vhostuserDev.QueueSize != 0 {
+		queueSize = vhostuserDev.QueueSize
+	}
+	deviceParams = append(deviceParams, fmt.Sprintf("queue-size=%d", queueSize))
 	if vhostuserDev.CacheSize != 0 {
 		deviceParams = append(deviceParams, fmt.Sprintf("cache-size=%dM", vhostuserDev.CacheSize))
 	}

src/runtime/pkg/katautils/config.go

@@ -114,6 +114,7 @@ type hypervisor struct {
 	RxRateLimiterMaxRate           uint64   `toml:"rx_rate_limiter_max_rate"`
 	TxRateLimiterMaxRate           uint64   `toml:"tx_rate_limiter_max_rate"`
 	MemOffset                      uint64   `toml:"memory_offset"`
+	DefaultMaxMemorySize           uint64   `toml:"default_maxmemory"`
 	DiskRateLimiterBwMaxRate       int64    `toml:"disk_rate_limiter_bw_max_rate"`
 	DiskRateLimiterBwOneTimeBurst  int64    `toml:"disk_rate_limiter_bw_one_time_burst"`
 	DiskRateLimiterOpsMaxRate      int64    `toml:"disk_rate_limiter_ops_max_rate"`
@@ -123,10 +124,10 @@ type hypervisor struct {
 	NetRateLimiterOpsMaxRate       int64    `toml:"net_rate_limiter_ops_max_rate"`
 	NetRateLimiterOpsOneTimeBurst  int64    `toml:"net_rate_limiter_ops_one_time_burst"`
 	VirtioFSCacheSize              uint32   `toml:"virtio_fs_cache_size"`
+	VirtioFSQueueSize              uint32   `toml:"virtio_fs_queue_size"`
 	DefaultMaxVCPUs                uint32   `toml:"default_maxvcpus"`
 	MemorySize                     uint32   `toml:"default_memory"`
 	MemSlots                       uint32   `toml:"memory_slots"`
-	DefaultMaxMemorySize           uint64   `toml:"default_maxmemory"`
 	DefaultBridges                 uint32   `toml:"default_bridges"`
 	Msize9p                        uint32   `toml:"msize_9p"`
 	PCIeRootPort                   uint32   `toml:"pcie_root_port"`
@@ -798,6 +799,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		VirtioFSDaemonList:      h.VirtioFSDaemonList,
 		VirtioFSCacheSize:       h.VirtioFSCacheSize,
 		VirtioFSCache:           h.defaultVirtioFSCache(),
+		VirtioFSQueueSize:       h.VirtioFSQueueSize,
 		VirtioFSExtraArgs:       h.VirtioFSExtraArgs,
 		MemPrealloc:             h.MemPrealloc,
 		HugePages:               h.HugePages,

src/runtime/virtcontainers/clh.go

@@ -1587,6 +1587,9 @@ func (clh *cloudHypervisor) addVolume(volume types.Volume) error {
 	// default values defined by cloud-hypervisor
 	numQueues := int32(1)
 	queueSize := int32(1024)
+	if clh.config.VirtioFSQueueSize != 0 {
+		queueSize = int32(clh.config.VirtioFSQueueSize)
+	}
 
 	fs := chclient.NewFsConfig(volume.MountTag, vfsdSockPath, numQueues, queueSize)
 	clh.vmconfig.Fs = &[]chclient.FsConfig{*fs}

src/runtime/virtcontainers/hypervisor.go

@@ -467,6 +467,9 @@ type HypervisorConfig struct {
 	// VirtioFSCacheSize is the DAX cache size in MiB
 	VirtioFSCacheSize uint32
 
+	// Size of virtqueues
+	VirtioFSQueueSize uint32
+
 	// User ID.
 	Uid uint32
 

src/runtime/virtcontainers/qemu.go

@@ -2080,6 +2080,7 @@ func (q *qemu) AddDevice(ctx context.Context, devInfo interface{}, devType Devic
 				Type:      config.VhostUserFS,
 				CacheSize: q.config.VirtioFSCacheSize,
 				Cache:     q.config.VirtioFSCache,
+				QueueSize: q.config.VirtioFSQueueSize,
 			}
 			vhostDev.SocketPath = sockPath
 			vhostDev.DevID = id

src/runtime/virtcontainers/qemu_arch_base.go

@@ -663,6 +663,7 @@ func (q *qemuArchBase) appendVhostUserDevice(ctx context.Context, devices []govm
 		qemuVhostUserDevice.TypeDevID = utils.MakeNameID("fs", attr.DevID, maxDevIDSize)
 		qemuVhostUserDevice.Tag = attr.Tag
 		qemuVhostUserDevice.CacheSize = attr.CacheSize
+		qemuVhostUserDevice.QueueSize = attr.QueueSize
 		qemuVhostUserDevice.VhostUserType = govmmQemu.VhostUserFS
 	}