From 66219d164f6b549b0ab416fc6afcb5028821a056 Mon Sep 17 00:00:00 2001
From: Julio Montes <julio.montes@intel.com>
Date: Mon, 29 Jun 2020 10:00:22 -0500
Subject: [PATCH] device: support vfio cold plug

Depending on ColdPlug flag, cold or hot plug vfio devices. The VFIO device
won't be hot removed when such flag is false

Signed-off-by: Julio Montes <julio.montes@intel.com>
---
 .../virtcontainers/device/drivers/vfio.go     | 27 ++++++++++++++++---
 src/runtime/virtcontainers/sandbox.go         |  9 +++++++
 2 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/src/runtime/virtcontainers/device/drivers/vfio.go b/src/runtime/virtcontainers/device/drivers/vfio.go
index ba18f0e3f3..5ce9d56b38 100644
--- a/src/runtime/virtcontainers/device/drivers/vfio.go
+++ b/src/runtime/virtcontainers/device/drivers/vfio.go
@@ -98,10 +98,20 @@ func (device *VFIODevice) Attach(devReceiver api.DeviceReceiver) (retErr error)
 		}
 	}
 
-	// hotplug a VFIO device is actually hotplugging a group of iommu devices
-	if err := devReceiver.HotplugAddDevice(device, config.DeviceVFIO); err != nil {
-		deviceLogger().WithError(err).Error("Failed to add device")
-		return err
+	coldPlug := device.DeviceInfo.ColdPlug
+	deviceLogger().WithField("cold-plug", coldPlug).Info("Attaching VFIO device")
+
+	if coldPlug {
+		if err := devReceiver.AppendDevice(device); err != nil {
+			deviceLogger().WithError(err).Error("Failed to append device")
+			return err
+		}
+	} else {
+		// hotplug a VFIO device is actually hotplugging a group of iommu devices
+		if err := devReceiver.HotplugAddDevice(device, config.DeviceVFIO); err != nil {
+			deviceLogger().WithError(err).Error("Failed to add device")
+			return err
+		}
 	}
 
 	deviceLogger().WithFields(logrus.Fields{
@@ -128,6 +138,15 @@ func (device *VFIODevice) Detach(devReceiver api.DeviceReceiver) (retErr error)
 		}
 	}()
 
+	if device.GenericDevice.DeviceInfo.ColdPlug {
+		// nothing to detach, device was cold plugged
+		deviceLogger().WithFields(logrus.Fields{
+			"device-group": device.DeviceInfo.HostPath,
+			"device-type":  "vfio-passthrough",
+		}).Info("Nothing to detach. VFIO device was cold plugged")
+		return nil
+	}
+
 	// hotplug a VFIO device is actually hotplugging a group of iommu devices
 	if err := devReceiver.HotplugRemoveDevice(device, config.DeviceVFIO); err != nil {
 		deviceLogger().WithError(err).Error("Failed to remove device")
diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go
index 8704d8b8ea..d5dae8a660 100644
--- a/src/runtime/virtcontainers/sandbox.go
+++ b/src/runtime/virtcontainers/sandbox.go
@@ -1713,7 +1713,16 @@ func (s *Sandbox) AppendDevice(device api.Device) error {
 	switch device.DeviceType() {
 	case config.VhostUserSCSI, config.VhostUserNet, config.VhostUserBlk, config.VhostUserFS:
 		return s.hypervisor.addDevice(device.GetDeviceInfo().(*config.VhostUserDeviceAttrs), vhostuserDev)
+	case config.DeviceVFIO:
+		vfioDevs := device.GetDeviceInfo().([]*config.VFIODev)
+		for _, d := range vfioDevs {
+			return s.hypervisor.addDevice(*d, vfioDev)
+		}
+	default:
+		s.Logger().WithField("device-type", device.DeviceType()).
+			Warn("Could not append device: unsupported device type")
 	}
+
 	return fmt.Errorf("unsupported device type")
 }