virtiofs: Add cache option

Several cache modes are supported by virtio-fs.  They affect the
performance and consistency characteristics of the file system.

For the time being cache="none" is recommended, but the other modes can
be experimented with.

Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>

Makefile

@@ -160,6 +160,7 @@ DEFSHAREDFS := virtio-9p
 DEFVIRTIOFSDAEMON :=
 # Default DAX mapping cache size in MiB
 DEFVIRTIOFSCACHESIZE := 8192
+DEFVIRTIOFSCACHE := always
 DEFENABLEIOTHREADS := false
 DEFENABLEMEMPREALLOC := false
 DEFENABLEHUGEPAGES := false
@@ -327,6 +328,7 @@ USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU
 USER_VARS += DEFSHAREDFS
 USER_VARS += DEFVIRTIOFSDAEMON
 USER_VARS += DEFVIRTIOFSCACHESIZE
+USER_VARS += DEFVIRTIOFSCACHE
 USER_VARS += DEFENABLEIOTHREADS
 USER_VARS += DEFENABLEMEMPREALLOC
 USER_VARS += DEFENABLEHUGEPAGES
@@ -466,6 +468,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit
 		-e "s|@DEFSHAREDFS@|$(DEFSHAREDFS)|g" \
 		-e "s|@DEFVIRTIOFSDAEMON@|$(DEFVIRTIOFSDAEMON)|g" \
 		-e "s|@DEFVIRTIOFSCACHESIZE@|$(DEFVIRTIOFSCACHESIZE)|g" \
+		-e "s|@DEFVIRTIOFSCACHE@|$(DEFVIRTIOFSCACHE)|g" \
 		-e "s|@DEFENABLEIOTHREADS@|$(DEFENABLEIOTHREADS)|g" \
 		-e "s|@DEFENABLEMEMPREALLOC@|$(DEFENABLEMEMPREALLOC)|g" \
 		-e "s|@DEFENABLEHUGEPAGES@|$(DEFENABLEHUGEPAGES)|g" \

cli/config/configuration-qemu.toml.in

@@ -108,6 +108,21 @@ virtio_fs_daemon = "@DEFVIRTIOFSDAEMON@"
 # Default size of DAX cache in MiB
 virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
 
+# Cache mode:
+#
+#  - none
+#    Metadata, data, and pathname lookup are not cached in guest. They are
+#    always fetched from host and any changes are immediately pushed to host.
+#
+#  - auto
+#    Metadata and pathname lookup cache expires after a configured amount of
+#    time (default is 1 second). Data is cached while the file is open (close
+#    to open consistency).
+#
+#  - always
+#    Metadata, data, and pathname lookup are cached in guest and never expire.
+virtio_fs_cache = "@DEFVIRTIOFSCACHE@"
+
 # Block storage driver to be used for the hypervisor in case the container
 # rootfs is backed by a block device. This is virtio-scsi, virtio-blk
 # or nvdimm.

pkg/katautils/config.go

@@ -94,6 +94,7 @@ type hypervisor struct {
 	EntropySource           string `toml:"entropy_source"`
 	SharedFS                string `toml:"shared_fs"`
 	VirtioFSDaemon          string `toml:"virtio_fs_daemon"`
+	VirtioFSCache           string `toml:"virtio_fs_cache"`
 	VirtioFSCacheSize       uint32 `toml:"virtio_fs_cache_size"`
 	BlockDeviceCacheSet     bool   `toml:"block_device_cache_set"`
 	BlockDeviceCacheDirect  bool   `toml:"block_device_cache_direct"`
@@ -580,6 +581,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		SharedFS:                sharedFS,
 		VirtioFSDaemon:          h.VirtioFSDaemon,
 		VirtioFSCacheSize:       h.VirtioFSCacheSize,
+		VirtioFSCache:           h.VirtioFSCache,
 		MemPrealloc:             h.MemPrealloc,
 		HugePages:               h.HugePages,
 		Mlock:                   !h.Swap,

virtcontainers/device/config/config.go

@@ -189,6 +189,7 @@ type VhostUserDeviceAttrs struct {
 	// These are only meaningful for vhost user fs devices
 	Tag       string
 	CacheSize uint32
+	Cache     string
 }
 
 // GetHostPathFunc is function pointer used to mock GetHostPath in tests.

virtcontainers/documentation/api/1.0/api.md

@@ -149,6 +149,9 @@ type HypervisorConfig struct {
 	// VirtioFSCacheSize is the virtio-fs DAX cache size in MiB
 	VirtioFSCacheSize uint32
 
+	// VirtioFSCache cache mode for fs version cache or "none"
+	VirtioFSCache string
+
 	// KernelParams are additional guest kernel parameters.
 	KernelParams []Param
 

virtcontainers/hypervisor.go

@@ -226,6 +226,9 @@ type HypervisorConfig struct {
 	// VirtioFSDaemon is the virtio-fs vhost-user daemon path
 	VirtioFSDaemon string
 
+	// VirtioFSCache cache mode for fs version cache or "none"
+	VirtioFSCache string
+
 	// customAssets is a map of assets.
 	// Each value in that map takes precedence over the configured assets.
 	// For example, if there is a value for the "kernel" key in this map,

virtcontainers/qemu.go

@@ -600,7 +600,8 @@ func (q *qemu) startSandbox(timeout int) error {
 		sourcePath := filepath.Join(kataHostSharedDir, q.id)
 		cmd := exec.Command(q.config.VirtioFSDaemon,
 			"-o", "vhost_user_socket="+sockPath,
-			"-o", "source="+sourcePath)
+			"-o", "source="+sourcePath,
+			"-o", "cache="+q.config.VirtioFSCache)
 		stderr, err := cmd.StderrPipe()
 		if err != nil {
 			return err
@@ -1379,6 +1380,7 @@ func (q *qemu) addDevice(devInfo interface{}, devType deviceType) error {
 				Tag:       v.MountTag,
 				Type:      config.VhostUserFS,
 				CacheSize: q.config.VirtioFSCacheSize,
+				Cache:     q.config.VirtioFSCache,
 			}
 			vhostDev.SocketPath = sockPath
 			vhostDev.DevID = id