From 47edcb3fec5021a166bcc69f74b60ef955b3c646 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Tue, 20 Mar 2018 12:21:46 -0600 Subject: [PATCH 1/2] virtcontainers: vfio: add support for hot plugging VFIO devices With this patch VFIO devices are hot plugged in the VM, that means no more cold plug in kata containers. fixes #85 Signed-off-by: Julio Montes --- virtcontainers/device.go | 8 +++++- virtcontainers/qemu.go | 53 +++++++++++++++++++++++++++++++++++----- 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/virtcontainers/device.go b/virtcontainers/device.go index ffe0be486e..e9943bdb2a 100644 --- a/virtcontainers/device.go +++ b/virtcontainers/device.go @@ -132,7 +132,13 @@ func (device *VFIODevice) attach(h hypervisor, c *Container) error { device.BDF = deviceBDF - if err := h.addDevice(*device, vfioDev); err != nil { + randBytes, err := generateRandomBytes(8) + if err != nil { + return err + } + device.DeviceInfo.ID = hex.EncodeToString(randBytes) + + if err := h.hotplugAddDevice(*device, vfioDev); err != nil { deviceLogger().WithError(err).Error("Failed to add device") return err } diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index c7025230a2..ce15db43e8 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -346,12 +346,12 @@ func (q *qemu) createPod(podConfig PodConfig) error { return err } - if q.config.BlockDeviceDriver == VirtioBlock { - devices = q.arch.appendBridges(devices, q.state.Bridges) - if err != nil { - return err - } - } else { + devices = q.arch.appendBridges(devices, q.state.Bridges) + if err != nil { + return err + } + + if q.config.BlockDeviceDriver == VirtioSCSI { devices = q.arch.appendSCSIController(devices) } @@ -638,6 +638,44 @@ func (q *qemu) hotplugBlockDevice(drive Drive, op operation) error { return nil } +func (q *qemu) hotplugVFIODevice(device VFIODevice, op operation) error { + defer func(qemu *qemu) { + if q.qmpMonitorCh.qmp != nil { + q.qmpMonitorCh.qmp.Shutdown() + } + }(q) + + qmp, err := q.qmpSetup() + if err != nil { + return err + } + + q.qmpMonitorCh.qmp = qmp + + devID := "vfio-" + device.DeviceInfo.ID + + if op == addDevice { + addr, bus, err := q.addDeviceToBridge(devID) + if err != nil { + return err + } + + if err := q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bus); err != nil { + return err + } + } else { + if err := q.removeDeviceFromBridge(devID); err != nil { + return err + } + + if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil { + return err + } + } + + return nil +} + func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operation) error { switch devType { case blockDev: @@ -646,6 +684,9 @@ func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operati case cpuDev: vcpus := devInfo.(uint32) return q.hotplugCPUs(vcpus, op) + case vfioDev: + device := devInfo.(VFIODevice) + return q.hotplugVFIODevice(device, op) default: return fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType) } From 249945f749c568a25233ae97e6118572107210ca Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Tue, 20 Mar 2018 12:42:00 -0600 Subject: [PATCH 2/2] virtcontainers: pod: remove cold plug support All devices are hot plugged in the VM, hence cold plug support is no more needed. Signed-off-by: Julio Montes --- virtcontainers/pod.go | 25 ------------------------- virtcontainers/pod_test.go | 4 ++-- 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/virtcontainers/pod.go b/virtcontainers/pod.go index e0779477be..0d1216f3c8 100644 --- a/virtcontainers/pod.go +++ b/virtcontainers/pod.go @@ -596,11 +596,6 @@ func createPod(podConfig PodConfig) (*Pod, error) { return nil, err } - // Passthrough devices - if err := p.attachDevices(); err != nil { - return nil, err - } - // Set pod state if err := p.setPodState(StateReady); err != nil { return nil, err @@ -1070,23 +1065,3 @@ func togglePausePod(podID string, pause bool) (*Pod, error) { return p, nil } - -func (p *Pod) attachDevices() error { - for _, container := range p.containers { - if err := container.attachDevices(); err != nil { - return err - } - } - - return nil -} - -func (p *Pod) detachDevices() error { - for _, container := range p.containers { - if err := container.detachDevices(); err != nil { - return err - } - } - - return nil -} diff --git a/virtcontainers/pod_test.go b/virtcontainers/pod_test.go index ff4e370d21..8acfaadf81 100644 --- a/virtcontainers/pod_test.go +++ b/virtcontainers/pod_test.go @@ -1171,10 +1171,10 @@ func TestPodAttachDevicesVFIO(t *testing.T) { containers[0].pod = &pod - err = pod.attachDevices() + err = containers[0].attachDevices() assert.Nil(t, err, "Error while attaching devices %s", err) - err = pod.detachDevices() + err = containers[0].detachDevices() assert.Nil(t, err, "Error while detaching devices %s", err) }