diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index 4afafaf08d..c91260a31a 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -25,9 +25,12 @@ image = "@IMAGEPATH@" # Known limitations: # * Does not work by design: # - CPU Hotplug -# - Device Hotplug # - Memory Hotplug # - NVDIMM devices +# - SharedFS, such as virtio-fs and virtio-fs-nydus +# +# Requirements: +# * virtio-block used as rootfs, thus the usage of devmapper snapshotter. # # Supported TEEs: # * Intel TDX diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index af5c583647..08d130815a 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -26,7 +26,6 @@ machine_type = "@MACHINETYPE@" # Known limitations: # * Does not work by design: # - CPU Hotplug -# - Device Hotplug # - Memory Hotplug # - NVDIMM devices # diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 805fda7169..c77ce15309 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -198,10 +198,125 @@ func (clh *cloudHypervisor) setConfig(config *HypervisorConfig) error { return nil } +func (clh *cloudHypervisor) createVirtiofsDaemon(sharedPath string) (VirtiofsDaemon, error) { + if !clh.supportsSharedFS() { + clh.Logger().Info("SharedFS is not supported") + return nil, nil + } + + virtiofsdSocketPath, err := clh.virtioFsSocketPath(clh.id) + if err != nil { + return nil, err + } + + if clh.config.SharedFS == config.VirtioFSNydus { + apiSockPath, err := clh.nydusdAPISocketPath(clh.id) + if err != nil { + clh.Logger().WithError(err).Error("Invalid api socket path for nydusd") + return nil, err + } + nd := &nydusd{ + path: clh.config.VirtioFSDaemon, + sockPath: virtiofsdSocketPath, + apiSockPath: apiSockPath, + sourcePath: sharedPath, + debug: clh.config.Debug, + extraArgs: clh.config.VirtioFSExtraArgs, + startFn: startInShimNS, + } + nd.setupShareDirFn = nd.setupPassthroughFS + return nd, nil + } + + // default: use virtiofsd + return &virtiofsd{ + path: clh.config.VirtioFSDaemon, + sourcePath: sharedPath, + socketPath: virtiofsdSocketPath, + extraArgs: clh.config.VirtioFSExtraArgs, + debug: clh.config.Debug, + cache: clh.config.VirtioFSCache, + }, nil +} + +func (clh *cloudHypervisor) setupVirtiofsDaemon(ctx context.Context) error { + if !clh.supportsSharedFS() { + clh.Logger().Info("SharedFS is not supported") + return nil + } + + if clh.config.SharedFS == config.Virtio9P { + return errors.New("cloud-hypervisor only supports virtio based file sharing") + } + + // virtioFS or virtioFsNydus + clh.Logger().WithField("function", "setupVirtiofsDaemon").Info("Starting virtiofsDaemon") + + if clh.virtiofsDaemon == nil { + return errors.New("Missing virtiofsDaemon configuration") + } + + pid, err := clh.virtiofsDaemon.Start(ctx, func() { + clh.StopVM(ctx, false) + }) + if err != nil { + return err + } + clh.state.VirtiofsDaemonPid = pid + + return nil +} + +func (clh *cloudHypervisor) stopVirtiofsDaemon(ctx context.Context) (err error) { + if !clh.supportsSharedFS() { + clh.Logger().Info("SharedFS is not supported") + return nil + } + + if clh.state.VirtiofsDaemonPid == 0 { + clh.Logger().Warn("The virtiofsd had stopped") + return nil + } + + err = clh.virtiofsDaemon.Stop(ctx) + if err != nil { + return err + } + + clh.state.VirtiofsDaemonPid = 0 + + return nil +} + +func (clh *cloudHypervisor) loadVirtiofsDaemon(sharedPath string) (VirtiofsDaemon, error) { + if !clh.supportsSharedFS() { + clh.Logger().Info("SharedFS is not supported") + return nil, nil + } + + virtiofsdSocketPath, err := clh.virtioFsSocketPath(clh.id) + if err != nil { + return nil, err + } + + return &virtiofsd{ + PID: clh.state.VirtiofsDaemonPid, + sourcePath: sharedPath, + debug: clh.config.Debug, + socketPath: virtiofsdSocketPath, + }, nil +} + func (clh *cloudHypervisor) nydusdAPISocketPath(id string) (string, error) { return utils.BuildSocketPath(clh.config.VMStorePath, id, nydusdAPISock) } +func (clh *cloudHypervisor) supportsSharedFS() bool { + caps := clh.Capabilities(clh.ctx) + + return caps.IsFsSharingSupported() +} + func (clh *cloudHypervisor) enableProtection() error { protection, err := availableGuestProtection() if err != nil { @@ -248,19 +363,15 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net clh.Logger().WithField("function", "CreateVM").Info("creating Sandbox") - virtiofsdSocketPath, err := clh.virtioFsSocketPath(clh.id) - if err != nil { - return nil - } - if clh.state.PID > 0 { clh.Logger().WithField("function", "CreateVM").Info("Sandbox already exist, loading from state") - clh.virtiofsDaemon = &virtiofsd{ - PID: clh.state.VirtiofsDaemonPid, - sourcePath: hypervisorConfig.SharedPath, - debug: clh.config.Debug, - socketPath: virtiofsdSocketPath, + + virtiofsDaemon, err := clh.loadVirtiofsDaemon(hypervisorConfig.SharedFS) + if err != nil { + return err } + clh.virtiofsDaemon = virtiofsDaemon + return nil } @@ -402,32 +513,9 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net ApiInternal: chclient.NewAPIClient(cfg).DefaultApi, } - clh.virtiofsDaemon = &virtiofsd{ - path: clh.config.VirtioFSDaemon, - sourcePath: filepath.Join(GetSharePath(clh.id)), - socketPath: virtiofsdSocketPath, - extraArgs: clh.config.VirtioFSExtraArgs, - debug: clh.config.Debug, - cache: clh.config.VirtioFSCache, - } - - if clh.config.SharedFS == config.VirtioFSNydus { - apiSockPath, err := clh.nydusdAPISocketPath(clh.id) - if err != nil { - clh.Logger().WithError(err).Error("Invalid api socket path for nydusd") - return err - } - nd := &nydusd{ - path: clh.config.VirtioFSDaemon, - sockPath: virtiofsdSocketPath, - apiSockPath: apiSockPath, - sourcePath: filepath.Join(GetSharePath(clh.id)), - debug: clh.config.Debug, - extraArgs: clh.config.VirtioFSExtraArgs, - startFn: startInShimNS, - } - nd.setupShareDirFn = nd.setupPassthroughFS - clh.virtiofsDaemon = nd + clh.virtiofsDaemon, err = clh.createVirtiofsDaemon(filepath.Join(GetSharePath(clh.id))) + if err != nil { + return err } if clh.config.SGXEPCSize > 0 { @@ -461,10 +549,6 @@ func (clh *cloudHypervisor) StartVM(ctx context.Context, timeout int) error { return err } - if clh.virtiofsDaemon == nil { - return errors.New("Missing virtiofsDaemon configuration") - } - // This needs to be done as late as possible, just before launching // virtiofsd are executed by kata-runtime after this call, run with // the SELinux label. If these processes require privileged, we do @@ -477,22 +561,14 @@ func (clh *cloudHypervisor) StartVM(ctx context.Context, timeout int) error { defer label.SetProcessLabel("") } - if clh.config.SharedFS == config.VirtioFS || clh.config.SharedFS == config.VirtioFSNydus { - clh.Logger().WithField("function", "StartVM").Info("Starting virtiofsDaemon") - pid, err := clh.virtiofsDaemon.Start(ctx, func() { - clh.StopVM(ctx, false) - }) - if err != nil { - return err - } - clh.state.VirtiofsDaemonPid = pid - } else { - return errors.New("cloud-hypervisor only supports virtio based file sharing") + err = clh.setupVirtiofsDaemon(ctx) + if err != nil { + return err } pid, err := clh.launchClh() if err != nil { - if shutdownErr := clh.virtiofsDaemon.Stop(ctx); shutdownErr != nil { + if shutdownErr := clh.stopVirtiofsDaemon(ctx); shutdownErr != nil { clh.Logger().WithError(shutdownErr).Warn("error shutting down VirtiofsDaemon") } return fmt.Errorf("failed to launch cloud-hypervisor: %q", err) @@ -638,10 +714,6 @@ func (clh *cloudHypervisor) HotplugAddDevice(ctx context.Context, devInfo interf span, _ := katatrace.Trace(ctx, clh.Logger(), "HotplugAddDevice", clhTracingTags, map[string]string{"sandbox_id": clh.id}) defer span.End() - if clh.config.ConfidentialGuest { - return nil, errors.New("Device hotplug addition is not supported in confidential mode") - } - switch devType { case BlockDev: drive := devInfo.(*config.BlockDrive) @@ -659,10 +731,6 @@ func (clh *cloudHypervisor) HotplugRemoveDevice(ctx context.Context, devInfo int span, _ := katatrace.Trace(ctx, clh.Logger(), "HotplugRemoveDevice", clhTracingTags, map[string]string{"sandbox_id": clh.id}) defer span.End() - if clh.config.ConfidentialGuest { - return nil, errors.New("Device hotplug removal is not supported in confidential mode") - } - var deviceID string switch devType { @@ -890,6 +958,10 @@ func (clh *cloudHypervisor) AddDevice(ctx context.Context, devInfo interface{}, case types.HybridVSock: clh.addVSock(defaultGuestVSockCID, v.UdsPath) case types.Volume: + if !clh.supportsSharedFS() { + return fmt.Errorf("SharedFS is not supported") + } + err = clh.addVolume(v) default: clh.Logger().WithField("function", "AddDevice").Warnf("Add device of type %v is not supported.", v) @@ -916,10 +988,10 @@ func (clh *cloudHypervisor) Capabilities(ctx context.Context) types.Capabilities clh.Logger().WithField("function", "Capabilities").Info("get Capabilities") var caps types.Capabilities - caps.SetFsSharingSupport() if !clh.config.ConfidentialGuest { - caps.SetBlockDeviceHotplugSupport() + caps.SetFsSharingSupport() } + caps.SetBlockDeviceHotplugSupport() return caps } @@ -957,12 +1029,9 @@ func (clh *cloudHypervisor) terminate(ctx context.Context, waitOnly bool) (err e return err } - if clh.virtiofsDaemon == nil { - return errors.New("virtiofsDaemon config is nil, failed to stop it") - } - clh.Logger().Debug("stop virtiofsDaemon") - if err = clh.virtiofsDaemon.Stop(ctx); err != nil { + + if err = clh.stopVirtiofsDaemon(ctx); err != nil { clh.Logger().WithError(err).Error("failed to stop virtiofsDaemon") } diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go index 8e76bf55c9..3f52ab756d 100644 --- a/src/runtime/virtcontainers/qemu_amd64.go +++ b/src/runtime/virtcontainers/qemu_amd64.go @@ -158,9 +158,8 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) { func (q *qemuAmd64) capabilities() types.Capabilities { var caps types.Capabilities - if (q.qemuMachine.Type == QemuQ35 || - q.qemuMachine.Type == QemuVirt) && - q.protection == noneProtection { + if q.qemuMachine.Type == QemuQ35 || + q.qemuMachine.Type == QemuVirt { caps.SetBlockDeviceHotplugSupport() } diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index c5c5e8057c..62fec60a7d 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -277,9 +277,7 @@ func (q *qemuArchBase) kernelParameters(debug bool) []Param { func (q *qemuArchBase) capabilities() types.Capabilities { var caps types.Capabilities - if q.protection == noneProtection { - caps.SetBlockDeviceHotplugSupport() - } + caps.SetBlockDeviceHotplugSupport() caps.SetMultiQueueSupport() caps.SetFsSharingSupport() return caps diff --git a/src/runtime/virtcontainers/qemu_ppc64le.go b/src/runtime/virtcontainers/qemu_ppc64le.go index e78fcb7016..e18f2264b5 100644 --- a/src/runtime/virtcontainers/qemu_ppc64le.go +++ b/src/runtime/virtcontainers/qemu_ppc64le.go @@ -101,8 +101,7 @@ func (q *qemuPPC64le) capabilities() types.Capabilities { var caps types.Capabilities // pseries machine type supports hotplugging drives - if q.qemuMachine.Type == QemuPseries && - q.protection == noneProtection { + if q.qemuMachine.Type == QemuPseries { caps.SetBlockDeviceHotplugSupport() }