github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-08-30 23:37:45 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #7953 from gkurz/extra-monitor-socket

Browse Source 
runtime/qemu: Rework QMP/HMP support
This commit is contained in:
Fabiano Fidêncio 2023-09-18 19:04:14 +02:00 committed by GitHub
commit c3ee913bf6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 217 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/runtime/config/configuration-qemu-nvidia-gpu.toml.in
View File

@ -323,11 +323,26 @@ valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@
pflashes = []
# This option changes the default hypervisor and kernel parameters
# to enable debug output where available. And Debug also enable the hmp socket.
# to enable debug output where available.
#
# Default false
#enable_debug = true
# This option allows to add an extra HMP or QMP socket when `enable_debug = true`
#
# WARNING: Anyone with access to the extra socket can take full control of
# Qemu. This is for debugging purpose only and must *NEVER* be used in
# production.
#
# Valid values are :
# - "hmp"
# - "qmp"
# - "qmp-pretty" (same as "qmp" with pretty json formatting)
#
# If set to the empty string "", no extra monitor socket is added. This is
# the default.
#extra_monitor_socket = hmp
# Disable the customizations done in the runtime when it detects
# that it is running on top a VMM. This will result in the runtime
# behaving as it would when running on bare metal.

17
src/runtime/config/configuration-qemu.toml.in
View File

@ -323,11 +323,26 @@ valid_file_mem_backends = @DEFVALIDFILEMEMBACKENDS@
pflashes = []
# This option changes the default hypervisor and kernel parameters
# to enable debug output where available. And Debug also enable the hmp socket.
# to enable debug output where available.
#
# Default false
#enable_debug = true
# This option allows to add an extra HMP or QMP socket when `enable_debug = true`
#
# WARNING: Anyone with access to the extra socket can take full control of
# Qemu. This is for debugging purpose only and must *NEVER* be used in
# production.
#
# Valid values are :
# - "hmp"
# - "qmp"
# - "qmp-pretty" (same as "qmp" with pretty json formatting)
#
# If set to the empty string "", no extra monitor socket is added. This is
# the default.
#extra_monitor_socket = hmp
# Disable the customizations done in the runtime when it detects
# that it is running on top a VMM. This will result in the runtime
# behaving as it would when running on bare metal.

31
src/runtime/pkg/govmm/qemu/qemu.go
View File

@ -2471,14 +2471,28 @@ const (
Unix QMPSocketType = "unix"
)
// QMPSocket represents a qemu QMP socket configuration.
// MonitorProtocol tells what protocol is used on a QMPSocket
type MonitorProtocol string
const (
// Socket using a human-friendly text-based protocol.
Hmp MonitorProtocol = "hmp"
// Socket using a richer json-based protocol.
Qmp MonitorProtocol = "qmp"
// Same as Qmp with pretty json formatting.
QmpPretty MonitorProtocol = "qmp-pretty"
)
// QMPSocket represents a qemu QMP or HMP socket configuration.
// nolint: govet
type QMPSocket struct {
// Type is the socket type (e.g. "unix").
Type QMPSocketType
// Human Monitor Interface (HMP) (true for HMP, false for QMP, default false)
IsHmp bool
// Protocol is the protocol to be used on the socket.
Protocol MonitorProtocol
// QMP listener file descriptor to be passed to qemu
FD *os.File
@ -2504,6 +2518,10 @@ func (qmp QMPSocket) Valid() bool {
return false
}
if qmp.Protocol != Hmp && qmp.Protocol != Qmp && qmp.Protocol != QmpPretty {
return false
}
return true
}
@ -2855,10 +2873,11 @@ func (config *Config) appendQMPSockets() {
}
}
if q.IsHmp {
switch q.Protocol {
case Hmp:
config.qemuParams = append(config.qemuParams, "-monitor")
} else {
config.qemuParams = append(config.qemuParams, "-qmp")
default:
config.qemuParams = append(config.qemuParams, fmt.Sprintf("-%s", q.Protocol))
}
config.qemuParams = append(config.qemuParams, strings.Join(qmpParams, ","))

43
src/runtime/pkg/govmm/qemu/qemu_test.go
View File

@ -726,10 +726,11 @@ var qmpSingleSocketString = "-qmp unix:path=cc-qmp"
func TestAppendSingleQMPSocketServer(t *testing.T) {
qmp := QMPSocket{
Type: "unix",
Name: "cc-qmp",
Server: true,
NoWait: true,
Type: "unix",
Name: "cc-qmp",
Server: true,
NoWait: true,
Protocol: Qmp,
}
testAppend(qmp, qmpSingleSocketServerString, t)
@ -737,9 +738,10 @@ func TestAppendSingleQMPSocketServer(t *testing.T) {
func TestAppendSingleQMPSocket(t *testing.T) {
qmp := QMPSocket{
Type: Unix,
Name: "cc-qmp",
Server: false,
Type: Unix,
Name: "cc-qmp",
Server: false,
Protocol: Qmp,
}
testAppend(qmp, qmpSingleSocketString, t)
@ -756,10 +758,11 @@ func TestAppendQMPSocketServerFd(t *testing.T) {
}()
qmp := QMPSocket{
Type: "unix",
FD: foo,
Server: true,
NoWait: true,
Type: "unix",
FD: foo,
Server: true,
NoWait: true,
Protocol: Qmp,
}
testAppend(qmp, qmpSocketServerFdString, t)
@ -770,16 +773,18 @@ var qmpSocketServerString = "-qmp unix:path=cc-qmp-1,server=on,wait=off -qmp uni
func TestAppendQMPSocketServer(t *testing.T) {
qmp := []QMPSocket{
{
Type: "unix",
Name: "cc-qmp-1",
Server: true,
NoWait: true,
Type: "unix",
Name: "cc-qmp-1",
Server: true,
NoWait: true,
Protocol: Qmp,
},
{
Type: "unix",
Name: "cc-qmp-2",
Server: true,
NoWait: true,
Type: "unix",
Name: "cc-qmp-2",
Server: true,
NoWait: true,
Protocol: Qmp,
},
}

2
src/runtime/pkg/katautils/config-settings.go.in
View File

@ -11,6 +11,7 @@ package katautils
import (
config "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
)
// name is the name of the runtime
@ -79,6 +80,7 @@ const defaultEnableIOMMU bool = false
const defaultEnableIOMMUPlatform bool = false
const defaultFileBackedMemRootDir string = ""
const defaultEnableDebug bool = false
const defaultExtraMonitorSocket govmmQemu.MonitorProtocol = ""
const defaultDisableNestingChecks bool = false
const defaultMsize9p uint32 = 8192
const defaultEntropySource = "/dev/urandom"

186
src/runtime/pkg/katautils/config.go
View File

@ -78,87 +78,88 @@ type factory struct {
}
type hypervisor struct {
Path string `toml:"path"`
JailerPath string `toml:"jailer_path"`
Kernel string `toml:"kernel"`
CtlPath string `toml:"ctlpath"`
Initrd string `toml:"initrd"`
Image string `toml:"image"`
RootfsType string `toml:"rootfs_type"`
Firmware string `toml:"firmware"`
FirmwareVolume string `toml:"firmware_volume"`
MachineAccelerators string `toml:"machine_accelerators"`
CPUFeatures string `toml:"cpu_features"`
KernelParams string `toml:"kernel_params"`
MachineType string `toml:"machine_type"`
BlockDeviceDriver string `toml:"block_device_driver"`
EntropySource string `toml:"entropy_source"`
SharedFS string `toml:"shared_fs"`
VirtioFSDaemon string `toml:"virtio_fs_daemon"`
VirtioFSCache string `toml:"virtio_fs_cache"`
VhostUserStorePath string `toml:"vhost_user_store_path"`
FileBackedMemRootDir string `toml:"file_mem_backend"`
GuestHookPath string `toml:"guest_hook_path"`
GuestMemoryDumpPath string `toml:"guest_memory_dump_path"`
SeccompSandbox string `toml:"seccompsandbox"`
BlockDeviceAIO string `toml:"block_device_aio"`
HypervisorPathList []string `toml:"valid_hypervisor_paths"`
JailerPathList []string `toml:"valid_jailer_paths"`
CtlPathList []string `toml:"valid_ctlpaths"`
VirtioFSDaemonList []string `toml:"valid_virtio_fs_daemon_paths"`
VirtioFSExtraArgs []string `toml:"virtio_fs_extra_args"`
PFlashList []string `toml:"pflashes"`
VhostUserStorePathList []string `toml:"valid_vhost_user_store_paths"`
FileBackedMemRootList []string `toml:"valid_file_mem_backends"`
EntropySourceList []string `toml:"valid_entropy_sources"`
EnableAnnotations []string `toml:"enable_annotations"`
RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"`
TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"`
MemOffset uint64 `toml:"memory_offset"`
DefaultMaxMemorySize uint64 `toml:"default_maxmemory"`
DiskRateLimiterBwMaxRate int64 `toml:"disk_rate_limiter_bw_max_rate"`
DiskRateLimiterBwOneTimeBurst int64 `toml:"disk_rate_limiter_bw_one_time_burst"`
DiskRateLimiterOpsMaxRate int64 `toml:"disk_rate_limiter_ops_max_rate"`
DiskRateLimiterOpsOneTimeBurst int64 `toml:"disk_rate_limiter_ops_one_time_burst"`
NetRateLimiterBwMaxRate int64 `toml:"net_rate_limiter_bw_max_rate"`
NetRateLimiterBwOneTimeBurst int64 `toml:"net_rate_limiter_bw_one_time_burst"`
NetRateLimiterOpsMaxRate int64 `toml:"net_rate_limiter_ops_max_rate"`
NetRateLimiterOpsOneTimeBurst int64 `toml:"net_rate_limiter_ops_one_time_burst"`
VirtioFSCacheSize uint32 `toml:"virtio_fs_cache_size"`
VirtioFSQueueSize uint32 `toml:"virtio_fs_queue_size"`
DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"`
MemorySize uint32 `toml:"default_memory"`
MemSlots uint32 `toml:"memory_slots"`
DefaultBridges uint32 `toml:"default_bridges"`
Msize9p uint32 `toml:"msize_9p"`
NumVCPUs int32 `toml:"default_vcpus"`
BlockDeviceCacheSet bool `toml:"block_device_cache_set"`
BlockDeviceCacheDirect bool `toml:"block_device_cache_direct"`
BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"`
EnableVhostUserStore bool `toml:"enable_vhost_user_store"`
VhostUserDeviceReconnect uint32 `toml:"vhost_user_reconnect_timeout_sec"`
DisableBlockDeviceUse bool `toml:"disable_block_device_use"`
MemPrealloc bool `toml:"enable_mem_prealloc"`
HugePages bool `toml:"enable_hugepages"`
VirtioMem bool `toml:"enable_virtio_mem"`
IOMMU bool `toml:"enable_iommu"`
IOMMUPlatform bool `toml:"enable_iommu_platform"`
Debug bool `toml:"enable_debug"`
DisableNestingChecks bool `toml:"disable_nesting_checks"`
EnableIOThreads bool `toml:"enable_iothreads"`
DisableImageNvdimm bool `toml:"disable_image_nvdimm"`
HotPlugVFIO config.PCIePort `toml:"hot_plug_vfio"`
ColdPlugVFIO config.PCIePort `toml:"cold_plug_vfio"`
DisableVhostNet bool `toml:"disable_vhost_net"`
GuestMemoryDumpPaging bool `toml:"guest_memory_dump_paging"`
ConfidentialGuest bool `toml:"confidential_guest"`
SevSnpGuest bool `toml:"sev_snp_guest"`
GuestSwap bool `toml:"enable_guest_swap"`
Rootless bool `toml:"rootless"`
DisableSeccomp bool `toml:"disable_seccomp"`
DisableSeLinux bool `toml:"disable_selinux"`
DisableGuestSeLinux bool `toml:"disable_guest_selinux"`
LegacySerial bool `toml:"use_legacy_serial"`
Path string `toml:"path"`
JailerPath string `toml:"jailer_path"`
Kernel string `toml:"kernel"`
CtlPath string `toml:"ctlpath"`
Initrd string `toml:"initrd"`
Image string `toml:"image"`
RootfsType string `toml:"rootfs_type"`
Firmware string `toml:"firmware"`
FirmwareVolume string `toml:"firmware_volume"`
MachineAccelerators string `toml:"machine_accelerators"`
CPUFeatures string `toml:"cpu_features"`
KernelParams string `toml:"kernel_params"`
MachineType string `toml:"machine_type"`
BlockDeviceDriver string `toml:"block_device_driver"`
EntropySource string `toml:"entropy_source"`
SharedFS string `toml:"shared_fs"`
VirtioFSDaemon string `toml:"virtio_fs_daemon"`
VirtioFSCache string `toml:"virtio_fs_cache"`
VhostUserStorePath string `toml:"vhost_user_store_path"`
FileBackedMemRootDir string `toml:"file_mem_backend"`
GuestHookPath string `toml:"guest_hook_path"`
GuestMemoryDumpPath string `toml:"guest_memory_dump_path"`
SeccompSandbox string `toml:"seccompsandbox"`
BlockDeviceAIO string `toml:"block_device_aio"`
HypervisorPathList []string `toml:"valid_hypervisor_paths"`
JailerPathList []string `toml:"valid_jailer_paths"`
CtlPathList []string `toml:"valid_ctlpaths"`
VirtioFSDaemonList []string `toml:"valid_virtio_fs_daemon_paths"`
VirtioFSExtraArgs []string `toml:"virtio_fs_extra_args"`
PFlashList []string `toml:"pflashes"`
VhostUserStorePathList []string `toml:"valid_vhost_user_store_paths"`
FileBackedMemRootList []string `toml:"valid_file_mem_backends"`
EntropySourceList []string `toml:"valid_entropy_sources"`
EnableAnnotations []string `toml:"enable_annotations"`
RxRateLimiterMaxRate uint64 `toml:"rx_rate_limiter_max_rate"`
TxRateLimiterMaxRate uint64 `toml:"tx_rate_limiter_max_rate"`
MemOffset uint64 `toml:"memory_offset"`
DefaultMaxMemorySize uint64 `toml:"default_maxmemory"`
DiskRateLimiterBwMaxRate int64 `toml:"disk_rate_limiter_bw_max_rate"`
DiskRateLimiterBwOneTimeBurst int64 `toml:"disk_rate_limiter_bw_one_time_burst"`
DiskRateLimiterOpsMaxRate int64 `toml:"disk_rate_limiter_ops_max_rate"`
DiskRateLimiterOpsOneTimeBurst int64 `toml:"disk_rate_limiter_ops_one_time_burst"`
NetRateLimiterBwMaxRate int64 `toml:"net_rate_limiter_bw_max_rate"`
NetRateLimiterBwOneTimeBurst int64 `toml:"net_rate_limiter_bw_one_time_burst"`
NetRateLimiterOpsMaxRate int64 `toml:"net_rate_limiter_ops_max_rate"`
NetRateLimiterOpsOneTimeBurst int64 `toml:"net_rate_limiter_ops_one_time_burst"`
VirtioFSCacheSize uint32 `toml:"virtio_fs_cache_size"`
VirtioFSQueueSize uint32 `toml:"virtio_fs_queue_size"`
DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"`
MemorySize uint32 `toml:"default_memory"`
MemSlots uint32 `toml:"memory_slots"`
DefaultBridges uint32 `toml:"default_bridges"`
Msize9p uint32 `toml:"msize_9p"`
NumVCPUs int32 `toml:"default_vcpus"`
BlockDeviceCacheSet bool `toml:"block_device_cache_set"`
BlockDeviceCacheDirect bool `toml:"block_device_cache_direct"`
BlockDeviceCacheNoflush bool `toml:"block_device_cache_noflush"`
EnableVhostUserStore bool `toml:"enable_vhost_user_store"`
VhostUserDeviceReconnect uint32 `toml:"vhost_user_reconnect_timeout_sec"`
DisableBlockDeviceUse bool `toml:"disable_block_device_use"`
MemPrealloc bool `toml:"enable_mem_prealloc"`
HugePages bool `toml:"enable_hugepages"`
VirtioMem bool `toml:"enable_virtio_mem"`
IOMMU bool `toml:"enable_iommu"`
IOMMUPlatform bool `toml:"enable_iommu_platform"`
Debug bool `toml:"enable_debug"`
DisableNestingChecks bool `toml:"disable_nesting_checks"`
EnableIOThreads bool `toml:"enable_iothreads"`
DisableImageNvdimm bool `toml:"disable_image_nvdimm"`
HotPlugVFIO config.PCIePort `toml:"hot_plug_vfio"`
ColdPlugVFIO config.PCIePort `toml:"cold_plug_vfio"`
DisableVhostNet bool `toml:"disable_vhost_net"`
GuestMemoryDumpPaging bool `toml:"guest_memory_dump_paging"`
ConfidentialGuest bool `toml:"confidential_guest"`
SevSnpGuest bool `toml:"sev_snp_guest"`
GuestSwap bool `toml:"enable_guest_swap"`
Rootless bool `toml:"rootless"`
DisableSeccomp bool `toml:"disable_seccomp"`
DisableSeLinux bool `toml:"disable_selinux"`
DisableGuestSeLinux bool `toml:"disable_guest_selinux"`
LegacySerial bool `toml:"use_legacy_serial"`
ExtraMonitorSocket govmmQemu.MonitorProtocol `toml:"extra_monitor_socket"`
}
type runtime struct {
@ -516,6 +517,22 @@ func (h hypervisor) blockDeviceAIO() (string, error) {
return "", fmt.Errorf("Invalid hypervisor block storage I/O mechanism %v specified (supported AIO: %v)", h.BlockDeviceAIO, supportedBlockAIO)
}
func (h hypervisor) extraMonitorSocket() (govmmQemu.MonitorProtocol, error) {
supportedExtraMonitor := []govmmQemu.MonitorProtocol{govmmQemu.Hmp, govmmQemu.Qmp, govmmQemu.QmpPretty}
if h.ExtraMonitorSocket == "" {
return "", nil
}
for _, extra := range supportedExtraMonitor {
if extra == h.ExtraMonitorSocket {
return extra, nil
}
}
return "", fmt.Errorf("Invalid hypervisor extra monitor socket %v specified (supported values: %v)", h.ExtraMonitorSocket, supportedExtraMonitor)
}
func (h hypervisor) sharedFS() (string, error) {
supportedSharedFS := []string{config.Virtio9P, config.VirtioFS, config.VirtioFSNydus, config.NoSharedFS}
@ -819,6 +836,11 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
rxRateLimiterMaxRate := h.getRxRateLimiterCfg()
txRateLimiterMaxRate := h.getTxRateLimiterCfg()
extraMonitorSocket, err := h.extraMonitorSocket()
if err != nil {
return vc.HypervisorConfig{}, err
}
return vc.HypervisorConfig{
HypervisorPath: hypervisor,
HypervisorPathList: h.HypervisorPathList,
@ -887,6 +909,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
LegacySerial: h.LegacySerial,
DisableSeLinux: h.DisableSeLinux,
DisableGuestSeLinux: h.DisableGuestSeLinux,
ExtraMonitorSocket: extraMonitorSocket,
}, nil
}
@ -1284,6 +1307,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig {
IOMMUPlatform: defaultEnableIOMMUPlatform,
FileBackedMemRootDir: defaultFileBackedMemRootDir,
Debug: defaultEnableDebug,
ExtraMonitorSocket: defaultExtraMonitorSocket,
DisableNestingChecks: defaultDisableNestingChecks,
BlockDeviceDriver: defaultBlockDeviceDriver,
BlockDeviceAIO: defaultBlockDeviceAIO,

6
src/runtime/virtcontainers/hypervisor.go
View File

@ -17,6 +17,7 @@ import (
"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
@ -575,7 +576,7 @@ type HypervisorConfig struct {
EnableIOThreads bool
// Debug changes the default hypervisor and kernel parameters to
// enable debug output where available. And Debug also enable the hmp socket.
// enable debug output where available.
Debug bool
// MemPrealloc specifies if the memory should be pre-allocated
@ -641,6 +642,9 @@ type HypervisorConfig struct {
// Use legacy serial for the guest console
LegacySerial bool
// ExtraMonitorSocket allows to add an extra HMP or QMP socket when the VMM is Qemu
ExtraMonitorSocket govmmQemu.MonitorProtocol
}
// vcpu mapping from vcpu number to thread number

41
src/runtime/virtcontainers/qemu.go
View File

@ -119,11 +119,11 @@ type qemu struct {
}
const (
consoleSocket = "console.sock"
qmpSocket = "qmp.sock"
hmpSocket = "hmp.sock"
vhostFSSocket = "vhost-fs.sock"
nydusdAPISock = "nydusd-api.sock"
consoleSocket = "console.sock"
qmpSocket = "qmp.sock"
extraMonitorSocket = "extra-monitor.sock"
vhostFSSocket = "vhost-fs.sock"
nydusdAPISock = "nydusd-api.sock"
// memory dump format will be set to elf
memoryDumpFormat = "elf"
@ -329,8 +329,8 @@ func (q *qemu) qmpSocketPath(id string) (string, error) {
return utils.BuildSocketPath(q.config.VMStorePath, id, qmpSocket)
}
func (q *qemu) hmpSocketPath(id string) (string, error) {
return utils.BuildSocketPath(q.config.VMStorePath, id, hmpSocket)
func (q *qemu) extraMonitorSocketPath(id string) (string, error) {
return utils.BuildSocketPath(q.config.VMStorePath, id, extraMonitorSocket)
}
func (q *qemu) getQemuMachine() (govmmQemu.Machine, error) {
@ -361,24 +361,31 @@ func (q *qemu) createQmpSocket() ([]govmmQemu.QMPSocket, error) {
var sockets []govmmQemu.QMPSocket
sockets = append(sockets, govmmQemu.QMPSocket{
Type: "unix",
Server: true,
NoWait: true,
Type: "unix",
Protocol: govmmQemu.Qmp,
Server: true,
NoWait: true,
})
if q.HypervisorConfig().Debug {
humanMonitorSockPath, err := q.hmpSocketPath(q.id)
// The extra monitor socket allows an external user to take full
// control on Qemu and silently break the VM in all possible ways.
// It should only ever be used for debugging purposes, hence the
// check on Debug.
if q.HypervisorConfig().Debug && q.config.ExtraMonitorSocket != "" {
extraMonitorSockPath, err := q.extraMonitorSocketPath(q.id)
if err != nil {
return nil, err
}
sockets = append(sockets, govmmQemu.QMPSocket{
Type: "unix",
IsHmp: true,
Name: humanMonitorSockPath,
Server: true,
NoWait: true,
Type: "unix",
Protocol: q.config.ExtraMonitorSocket,
Name: extraMonitorSockPath,
Server: true,
NoWait: true,
})
q.Logger().Warn("QEMU configured to start with an untrusted monitor")
}
return sockets, nil