From a61fba6d457f67a1dda2983c28584decf3abf5b3 Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Wed, 23 Nov 2022 16:02:01 +0100 Subject: [PATCH] runtime: Pre-establish the QMP connection Running QEMU daemonized ensures that the QMP socket is ready to accept connections when LaunchQemu() returns. In order to be able to run QEMU undaemonized, let's handle that part upfront. Create a listener socket and connect to it. Pass the listener to QEMU and pass the connected socket to QMP : this ensures that we cannot fail to establish QMP connection and that we can detect if QEMU exits before accepting the connection. This is basically what libvirt does. Signed-off-by: Greg Kurz (cherry picked from commit 8a1723a5cb97aa69230864bd0435f2f21c198170) Signed-off-by: Greg Kurz --- src/runtime/virtcontainers/qemu.go | 67 ++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 785aa0182c..7ad589c641 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -15,6 +15,7 @@ import ( "encoding/json" "fmt" "math" + "net" "os" "os/user" "path/filepath" @@ -359,7 +360,6 @@ func (q *qemu) createQmpSocket() ([]govmmQemu.QMPSocket, error) { return []govmmQemu.QMPSocket{ { Type: "unix", - Name: q.qmpMonitorCh.path, Server: true, NoWait: true, }, @@ -796,6 +796,59 @@ func (q *qemu) setupVirtioMem(ctx context.Context) error { return err } +// setupEarlyQmpConnection creates a listener socket to be passed to QEMU +// as a QMP listening endpoint. An initial connection is established, to +// be used as the QMP client socket. This allows to detect an early failure +// of QEMU instead of looping on connect until some timeout expires. +func (q *qemu) setupEarlyQmpConnection() (net.Conn, error) { + monitorSockPath := q.qmpMonitorCh.path + + qmpListener, err := net.Listen("unix", monitorSockPath) + if err != nil { + q.Logger().WithError(err).Errorf("Unable to listen on unix socket address (%s)", monitorSockPath) + return nil, err + } + + // A duplicate fd of this socket will be passed to QEMU. We must + // close the original one when we're done. + defer qmpListener.Close() + + if rootless.IsRootless() { + err = syscall.Chown(monitorSockPath, int(q.config.Uid), int(q.config.Gid)) + if err != nil { + q.Logger().WithError(err).Errorf("Unable to make unix socket (%s) rootless", monitorSockPath) + return nil, err + } + } + + VMFd, err := qmpListener.(*net.UnixListener).File() + if err != nil { + return nil, err + } + defer func() { + if err != nil { + VMFd.Close() + } + }() + + // This socket will be used to establish the initial QMP connection + dialer := net.Dialer{Cancel: q.qmpMonitorCh.ctx.Done()} + conn, err := dialer.Dial("unix", monitorSockPath) + if err != nil { + q.Logger().WithError(err).Errorf("Unable to connect to unix socket (%s)", monitorSockPath) + return nil, err + } + + // We need to keep the socket file around to be able to re-connect + qmpListener.(*net.UnixListener).SetUnlinkOnClose(false) + + // Pass the duplicated fd of the listener socket to QEMU + q.qemuConfig.QMPSockets[0].FD = VMFd + q.fds = append(q.fds, q.qemuConfig.QMPSockets[0].FD) + + return conn, nil +} + // StartVM will start the Sandbox's VM. func (q *qemu) StartVM(ctx context.Context, timeout int) error { span, ctx := katatrace.Trace(ctx, q.Logger(), "StartVM", qemuTracingTags, map[string]string{"sandbox_id": q.id}) @@ -841,6 +894,12 @@ func (q *qemu) StartVM(ctx context.Context, timeout int) error { } }() + var qmpConn net.Conn + qmpConn, err = q.setupEarlyQmpConnection() + if err != nil { + return err + } + // This needs to be done as late as possible, just before launching // virtiofsd are executed by kata-runtime after this call, run with // the SELinux label. If these processes require privileged, we do @@ -880,7 +939,7 @@ func (q *qemu) StartVM(ctx context.Context, timeout int) error { return fmt.Errorf("failed to launch qemu: %s, error messages from qemu log: %s", err, strErr) } - err = q.waitVM(ctx, timeout) + err = q.waitVM(ctx, qmpConn, timeout) if err != nil { return err } @@ -918,7 +977,7 @@ func (q *qemu) bootFromTemplate() error { } // waitVM will wait for the Sandbox's VM to be up and running. -func (q *qemu) waitVM(ctx context.Context, timeout int) error { +func (q *qemu) waitVM(ctx context.Context, qmpConn net.Conn, timeout int) error { span, _ := katatrace.Trace(ctx, q.Logger(), "waitVM", qemuTracingTags, map[string]string{"sandbox_id": q.id}) defer span.End() @@ -938,7 +997,7 @@ func (q *qemu) waitVM(ctx context.Context, timeout int) error { timeStart := time.Now() for { disconnectCh = make(chan struct{}) - qmp, ver, err = govmmQemu.QMPStart(q.qmpMonitorCh.ctx, q.qmpMonitorCh.path, cfg, disconnectCh) + qmp, ver, err = govmmQemu.QMPStartWithConn(q.qmpMonitorCh.ctx, qmpConn, cfg, disconnectCh) if err == nil { break }