runtime: support non-root for clh

This change enables to run cloud-hypervisor VMM using a non-root user
when rootless flag is set true in the configuration

Fixes: #2567

Signed-off-by: Feng Wang <fwang@confluent.io>

docs/how-to/how-to-run-rootless-vmm.md

@@ -1,5 +1,5 @@
 ## Introduction
-To improve security, Kata Container supports running the VMM process (currently only QEMU) as a non-`root` user.
+To improve security, Kata Container supports running the VMM process (QEMU and cloud-hypervisor) as a non-`root` user. 
 This document describes how to enable the rootless VMM mode and its limitations.
 
 ## Pre-requisites
@@ -27,7 +27,7 @@ Another necessary change is to move the hypervisor runtime files (e.g. `vhost-fs
 ## Limitations
 
 1. Only the VMM process is running as a non-root user. Other processes such as Kata Container shimv2 and `virtiofsd` still run as the root user.
-2. Currently, this feature is only supported in QEMU. Still need to bring it to Firecracker and Cloud Hypervisor (see https://github.com/kata-containers/kata-containers/issues/2567).
+2. Currently, this feature is only supported in QEMU and cloud-hypervisor. For firecracker, you can use jailer to run the VMM process with a non-root user.
 3. Certain features will not work when rootless VMM is enabled, including:
    1. Passing devices to the guest (`virtio-blk`, `virtio-scsi`) will not work if the non-privileged user does not have permission to access it (leading to a permission denied error). A more permissive permission (e.g. 666) may overcome this issue. However, you need to be aware of the potential security implications of reducing the security on such devices.
    2. `vfio` device will also not work because of permission denied error.

src/runtime/config/configuration-clh.toml.in

@@ -41,6 +41,11 @@ rootfs_type=@DEFROOTFSTYPE@
 # Default false
 # confidential_guest = true
 
+# Enable running clh VMM as a non-root user.
+# By default clh VMM run as root. When this is set to true, clh VMM process runs as
+# a non-root random user. See documentation for the limitations of this mode.
+# rootless = true
+
 # disable applying SELinux on the VMM process (default false)
 disable_selinux=@DEFDISABLESELINUX@
 

src/runtime/pkg/katautils/config.go

@@ -1046,6 +1046,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		EnableAnnotations:              h.EnableAnnotations,
 		DisableSeccomp:                 h.DisableSeccomp,
 		ConfidentialGuest:              h.ConfidentialGuest,
+		Rootless:                       h.Rootless,
 		DisableSeLinux:                 h.DisableSeLinux,
 		DisableGuestSeLinux:            h.DisableGuestSeLinux,
 		NetRateLimiterBwMaxRate:        h.getNetRateLimiterBwMaxRate(),

src/runtime/virtcontainers/clh.go

@@ -19,6 +19,7 @@ import (
 	"net/http/httputil"
 	"os"
 	"os/exec"
+	"os/user"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -37,6 +38,8 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
+	pkgUtils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
@@ -653,7 +656,7 @@ func (clh *cloudHypervisor) StartVM(ctx context.Context, timeout int) error {
 	clh.Logger().WithField("function", "StartVM").Info("starting Sandbox")
 
 	vmPath := filepath.Join(clh.config.VMStorePath, clh.id)
-	err := os.MkdirAll(vmPath, DirMode)
+	err := utils.MkdirAllWithInheritedOwner(vmPath, DirMode)
 	if err != nil {
 		return err
 	}
@@ -1352,9 +1355,16 @@ func (clh *cloudHypervisor) launchClh() (int, error) {
 			cmdHypervisor.Stdout = clh.console
 		}
 	}
-
 	cmdHypervisor.Stderr = cmdHypervisor.Stdout
 
+	attr := syscall.SysProcAttr{}
+	attr.Credential = &syscall.Credential{
+		Uid:    clh.config.Uid,
+		Gid:    clh.config.Gid,
+		Groups: clh.config.Groups,
+	}
+	cmdHypervisor.SysProcAttr = &attr
+
 	err = utils.StartCmd(cmdHypervisor)
 	if err != nil {
 		return -1, err
@@ -1679,6 +1689,30 @@ func (clh *cloudHypervisor) cleanupVM(force bool) error {
 			clh.Logger().WithError(err).WithField("path", dir).Warnf("failed to remove vm path")
 		}
 	}
+	if rootless.IsRootless() {
+		if _, err := user.Lookup(clh.config.User); err != nil {
+			clh.Logger().WithError(err).WithFields(
+				log.Fields{
+					"user": clh.config.User,
+					"uid":  clh.config.Uid,
+				}).Warn("failed to find the user, it might have been removed")
+			return nil
+		}
+
+		if err := pkgUtils.RemoveVmmUser(clh.config.User); err != nil {
+			clh.Logger().WithError(err).WithFields(
+				log.Fields{
+					"user": clh.config.User,
+					"uid":  clh.config.Uid,
+				}).Warn("failed to delete the user")
+			return nil
+		}
+		clh.Logger().WithFields(
+			log.Fields{
+				"user": clh.config.User,
+				"uid":  clh.config.Uid,
+			}).Debug("successfully removed the non root user")
+	}
 
 	clh.reset()
 

src/runtime/virtcontainers/qemu.go

@@ -1183,6 +1183,7 @@ func (q *qemu) cleanupVM() error {
 					"user": q.config.User,
 					"uid":  q.config.Uid,
 				}).Warn("failed to delete the user")
+			return nil
 		}
 		q.Logger().WithFields(
 			logrus.Fields{