From 0de7572f7baf6328fb663f7a297e0977c89f0dd7 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Tue, 2 Oct 2018 17:09:03 -0700 Subject: [PATCH 1/2] vendor: Update govmm vendoring Shortlog: ef72505 qemu: Fix the support of PCIe bridge 56f645e qmp: add ExecuteQueryMigration a429677 govmm: fix memory prealloc 1130aab qmp: add "query-cpus" support Signed-off-by: Sebastien Boeuf --- Gopkg.lock | 4 +- Gopkg.toml | 2 +- vendor/github.com/intel/govmm/qemu/qemu.go | 25 ++-- vendor/github.com/intel/govmm/qemu/qmp.go | 132 ++++++++++++++++++++- 4 files changed, 147 insertions(+), 16 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index f63d494699..c396dcd6e7 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -123,11 +123,11 @@ revision = "3520598351bb3500a49ae9563f5539666ae0a27c" [[projects]] - digest = "1:034b5648db9f53a2b6f4f84925ee031cbeaba69daa10d95e74e8242707d7a5b0" + digest = "1:c63a5bf4f3fd94ae838ce53e3492c0a467c40d09cada9301d228df7164ba4e55" name = "github.com/intel/govmm" packages = ["qemu"] pruneopts = "NUT" - revision = "9905ae92c5915c07abeb669eaa4d7f7408834b51" + revision = "f03df80fc3dc52f65ed6c7d12806279b44185d32" [[projects]] digest = "1:672470f31bc4e50f9ba09a1af7ab6035bf8b1452db64dfd79b1a22614bb30710" diff --git a/Gopkg.toml b/Gopkg.toml index eb4723c046..9329c02b20 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -52,7 +52,7 @@ [[constraint]] name = "github.com/intel/govmm" - revision = "9905ae92c5915c07abeb669eaa4d7f7408834b51" + revision = "f03df80fc3dc52f65ed6c7d12806279b44185d32" [[constraint]] name = "github.com/kata-containers/agent" diff --git a/vendor/github.com/intel/govmm/qemu/qemu.go b/vendor/github.com/intel/govmm/qemu/qemu.go index c883de53a3..a44cc63f4a 100644 --- a/vendor/github.com/intel/govmm/qemu/qemu.go +++ b/vendor/github.com/intel/govmm/qemu/qemu.go @@ -938,18 +938,19 @@ func (bridgeDev BridgeDevice) Valid() bool { // QemuParams returns the qemu parameters built out of this bridge device. func (bridgeDev BridgeDevice) QemuParams(config *Config) []string { var qemuParams []string + var deviceParam string - shpc := "off" - if bridgeDev.SHPC { - shpc = "on" + switch bridgeDev.Type { + case PCIEBridge: + deviceParam = fmt.Sprintf("pcie-pci-bridge,bus=%s,id=%s", bridgeDev.Bus, bridgeDev.ID) + default: + shpc := "off" + if bridgeDev.SHPC { + shpc = "on" + } + deviceParam = fmt.Sprintf("pci-bridge,bus=%s,id=%s,chassis_nr=%d,shpc=%s", bridgeDev.Bus, bridgeDev.ID, bridgeDev.Chassis, shpc) } - deviceName := "pci-bridge" - if bridgeDev.Type == PCIEBridge { - deviceName = "pcie-pci-bridge" - } - - deviceParam := fmt.Sprintf("%s,bus=%s,id=%s,chassis_nr=%d,shpc=%s", deviceName, bridgeDev.Bus, bridgeDev.ID, bridgeDev.Chassis, shpc) if bridgeDev.Addr != "" { addr, err := strconv.Atoi(bridgeDev.Addr) if err == nil && addr >= 0 { @@ -1562,13 +1563,13 @@ func (config *Config) appendMemoryKnobs() { if config.Memory.Size != "" { dimmName := "dimm1" objMemParam := "memory-backend-ram,id=" + dimmName + ",size=" + config.Memory.Size + ",prealloc=on" - deviceMemParam := "pc-dimm,id=" + dimmName + ",memdev=" + dimmName + numaMemParam := "node,memdev=" + dimmName config.qemuParams = append(config.qemuParams, "-object") config.qemuParams = append(config.qemuParams, objMemParam) - config.qemuParams = append(config.qemuParams, "-device") - config.qemuParams = append(config.qemuParams, deviceMemParam) + config.qemuParams = append(config.qemuParams, "-numa") + config.qemuParams = append(config.qemuParams, numaMemParam) } } else if config.Knobs.FileBackedMem == true { if config.Memory.Size != "" && config.Memory.Path != "" { diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go index 9f764f1236..58cd76709c 100644 --- a/vendor/github.com/intel/govmm/qemu/qmp.go +++ b/vendor/github.com/intel/govmm/qemu/qmp.go @@ -144,7 +144,7 @@ type QMPVersion struct { Capabilities []string } -// CPUProperties contains the properties to be used for hotplugging a CPU instance +// CPUProperties contains the properties of a CPU instance type CPUProperties struct { Node int `json:"node-id"` Socket int `json:"socket-id"` @@ -178,6 +178,68 @@ type MemoryDevices struct { Type string `json:"type"` } +// CPUInfo represents information about each virtual CPU +type CPUInfo struct { + CPU int `json:"CPU"` + Current bool `json:"current"` + Halted bool `json:"halted"` + QomPath string `json:"qom_path"` + Arch string `json:"arch"` + Pc int `json:"pc"` + ThreadID int `json:"thread_id"` + Props CPUProperties `json:"props"` +} + +// CPUInfoFast represents information about each virtual CPU +type CPUInfoFast struct { + CPUIndex int `json:"cpu-index"` + QomPath string `json:"qom-path"` + Arch string `json:"arch"` + ThreadID int `json:"thread-id"` + Target string `json:"target"` + Props CPUProperties `json:"props"` +} + +// MigrationRAM represents migration ram status +type MigrationRAM struct { + Total int64 `json:"total"` + Remaining int64 `json:"remaining"` + Transferred int64 `json:"transferred"` + TotalTime int64 `json:"total-time"` + SetupTime int64 `json:"setup-time"` + ExpectedDowntime int64 `json:"expected-downtime"` + Duplicate int64 `json:"duplicate"` + Normal int64 `json:"normal"` + NormalBytes int64 `json:"normal-bytes"` + DirtySyncCount int64 `json:"dirty-sync-count"` +} + +// MigrationDisk represents migration disk status +type MigrationDisk struct { + Total int64 `json:"total"` + Remaining int64 `json:"remaining"` + Transferred int64 `json:"transferred"` +} + +// MigrationXbzrleCache represents migration XbzrleCache status +type MigrationXbzrleCache struct { + CacheSize int64 `json:"cache-size"` + Bytes int64 `json:"bytes"` + Pages int64 `json:"pages"` + CacheMiss int64 `json:"cache-miss"` + CacheMissRate int64 `json:"cache-miss-rate"` + Overflow int64 `json:"overflow"` +} + +// MigrationStatus represents migration status of a vm +type MigrationStatus struct { + Status string `json:"status"` + Capabilities []map[string]interface{} `json:"capabilities,omitempty"` + RAM MigrationRAM `json:"ram,omitempty"` + Disk MigrationDisk `json:"disk,omitempty"` + XbzrleCache MigrationXbzrleCache `json:"xbzrle-cache,omitempty"` +} + func (q *QMP) readLoop(fromVMCh chan<- []byte) { scanner := bufio.NewScanner(q.conn) for scanner.Scan() { @@ -1033,6 +1095,54 @@ func (q *QMP) ExecQueryMemoryDevices(ctx context.Context) ([]MemoryDevices, erro return memoryDevices, nil } +// ExecQueryCpus returns a slice with the list of `CpuInfo` +// Since qemu 2.12, we have `query-cpus-fast` as a better choice in production +// we can still choose `ExecQueryCpus` for compatibility though not recommended. +func (q *QMP) ExecQueryCpus(ctx context.Context) ([]CPUInfo, error) { + response, err := q.executeCommandWithResponse(ctx, "query-cpus", nil, nil, nil) + if err != nil { + return nil, err + } + + // convert response to json + data, err := json.Marshal(response) + if err != nil { + return nil, fmt.Errorf("Unable to extract memory devices information: %v", err) + } + + var cpuInfo []CPUInfo + // convert json to []CPUInfo + if err = json.Unmarshal(data, &cpuInfo); err != nil { + return nil, fmt.Errorf("unable to convert json to CPUInfo: %v", err) + } + + return cpuInfo, nil +} + +// ExecQueryCpusFast returns a slice with the list of `CpuInfoFast` +// This is introduced since 2.12, it does not incur a performance penalty and +// should be used in production instead of query-cpus. +func (q *QMP) ExecQueryCpusFast(ctx context.Context) ([]CPUInfoFast, error) { + response, err := q.executeCommandWithResponse(ctx, "query-cpus-fast", nil, nil, nil) + if err != nil { + return nil, err + } + + // convert response to json + data, err := json.Marshal(response) + if err != nil { + return nil, fmt.Errorf("Unable to extract memory devices information: %v", err) + } + + var cpuInfoFast []CPUInfoFast + // convert json to []CPUInfoFast + if err = json.Unmarshal(data, &cpuInfoFast); err != nil { + return nil, fmt.Errorf("unable to convert json to CPUInfoFast: %v", err) + } + + return cpuInfoFast, nil +} + // ExecHotplugMemory adds size of MiB memory to the guest func (q *QMP) ExecHotplugMemory(ctx context.Context, qomtype, id, mempath string, size int) error { args := map[string]interface{}{ @@ -1136,3 +1246,23 @@ func (q *QMP) ExecuteVirtSerialPortAdd(ctx context.Context, id, name, chardev st return q.executeCommand(ctx, "device_add", args, nil) } + +// ExecuteQueryMigration queries migration progress. +func (q *QMP) ExecuteQueryMigration(ctx context.Context) (MigrationStatus, error) { + response, err := q.executeCommandWithResponse(ctx, "query-migrate", nil, nil, nil) + if err != nil { + return MigrationStatus{}, err + } + + data, err := json.Marshal(response) + if err != nil { + return MigrationStatus{}, fmt.Errorf("Unable to extract migrate status information: %v", err) + } + + var status MigrationStatus + if err = json.Unmarshal(data, &status); err != nil { + return MigrationStatus{}, fmt.Errorf("Unable to convert migrate status information: %v", err) + } + + return status, nil +} From dffb4f96aedcbd5d412a562034f3854c48a6f9c8 Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Tue, 2 Oct 2018 12:13:18 -0700 Subject: [PATCH 2/2] virtcontainers: qemu: Add proper support for virt machine type The virt machine type provided by the NEMU project needs to be supported the same way we support pc and q35 machine types. First, this patch takes care of adding the hotpluggable block device capability to this machine type, this way when using devicemapper, we prevent the code from falling back on using 9pfs instead of SCSI. It also add one or several bridges to this machine type, as the code is tightly coupled to the fact that a bridge is required for PCI hotplug. At last, it changes the name of the PCI host bridge (main bus), to use "pcie.0". The default set up from pc machine type "pci.0" is not suitable for this machine type. Fixes #804 Signed-off-by: Sebastien Boeuf --- virtcontainers/qemu.go | 6 ++++-- virtcontainers/qemu_amd64.go | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 920ed9268b..fcaf70a838 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -1194,7 +1194,8 @@ func (q *qemu) disconnect() { // genericAppendBridges appends to devices the given bridges func genericAppendBridges(devices []govmmQemu.Device, bridges []Bridge, machineType string) []govmmQemu.Device { bus := defaultPCBridgeBus - if machineType == QemuQ35 { + switch machineType { + case QemuQ35, QemuVirt: bus = defaultBridgeBus } @@ -1227,13 +1228,14 @@ func genericBridges(number uint32, machineType string) []Bridge { var bt bridgeType switch machineType { - case QemuQ35: // currently only pci bridges are supported // qemu-2.10 will introduce pcie bridges fallthrough case QemuPC: bt = pciBridge + case QemuVirt: + bt = pcieBridge case QemuPseries: bt = pciBridge default: diff --git a/virtcontainers/qemu_amd64.go b/virtcontainers/qemu_amd64.go index 12089516d8..5eb002ff26 100644 --- a/virtcontainers/qemu_amd64.go +++ b/virtcontainers/qemu_amd64.go @@ -102,7 +102,9 @@ func newQemuArch(config HypervisorConfig) qemuArch { func (q *qemuAmd64) capabilities() capabilities { var caps capabilities - if q.machineType == QemuPC || q.machineType == QemuQ35 { + if q.machineType == QemuPC || + q.machineType == QemuQ35 || + q.machineType == QemuVirt { caps.setBlockDeviceHotplugSupport() }