gpu: Add configuration option for cold-plug VFIO

Users can set cold-plug="root-port" to cold plug a VFIO device in QEMU

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>

src/runtime/cmd/kata-runtime/kata-env.go

@@ -17,6 +17,7 @@ import (
 	"github.com/prometheus/procfs"
 	"github.com/urfave/cli"
 
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
@@ -113,6 +114,7 @@ type HypervisorInfo struct {
 	Msize9p              uint32
 	MemorySlots          uint32
 	PCIeRootPort         uint32
+	ColdPlugVFIO         hv.PCIePort
 	HotplugVFIOOnRootBus bool
 	Debug                bool
 }

src/runtime/cmd/kata-runtime/kata-env_test.go

@@ -24,6 +24,7 @@ import (
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/urfave/cli"
 
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
@@ -85,6 +86,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 	blockStorageDriver := "virtio-scsi"
 	enableIOThreads := true
 	hotplugVFIOOnRootBus := true
+	coldPlugVFIO := hv.RootPort
 	pcieRootPort := uint32(2)
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
@@ -129,6 +131,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 		BlockDeviceDriver:    blockStorageDriver,
 		EnableIOThreads:      enableIOThreads,
 		HotplugVFIOOnRootBus: hotplugVFIOOnRootBus,
+		ColdPlugVFIO:         coldPlugVFIO,
 		PCIeRootPort:         pcieRootPort,
 		DisableNewNetNs:      disableNewNetNs,
 		DefaultVCPUCount:     hypConfig.NumVCPUs,

src/runtime/pkg/katatestutils/utils.go

@@ -14,6 +14,7 @@ import (
 	"strconv"
 	"testing"
 
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"
 )
@@ -224,6 +225,7 @@ type RuntimeConfigOptions struct {
 	JaegerPassword       string
 	PFlash               []string
 	PCIeRootPort         uint32
+	ColdPlugVFIO         hv.PCIePort
 	DefaultVCPUCount     uint32
 	DefaultMaxVCPUCount  uint32
 	DefaultMemSize       uint32

src/runtime/pkg/katautils/config-settings.go.in

@@ -103,3 +103,5 @@ const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock"
 
 // Default config file used by stateless systems.
 var defaultRuntimeConfiguration = "@CONFIG_PATH@"
+
+const defaultColdPlugVFIO = "no-port"

src/runtime/pkg/katautils/config.go

@@ -20,6 +20,7 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
 	govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
@@ -148,6 +149,7 @@ type hypervisor struct {
 	EnableIOThreads                bool        `toml:"enable_iothreads"`
 	DisableImageNvdimm             bool        `toml:"disable_image_nvdimm"`
 	HotplugVFIOOnRootBus           bool        `toml:"hotplug_vfio_on_root_bus"`
+	ColdPlugVFIO                   hv.PCIePort `toml:"cold_plug_vfio"`
 	DisableVhostNet                bool        `toml:"disable_vhost_net"`
 	GuestMemoryDumpPaging          bool        `toml:"guest_memory_dump_paging"`
 	ConfidentialGuest              bool        `toml:"confidential_guest"`

src/runtime/virtcontainers/hypervisor.go

@@ -509,6 +509,10 @@ type HypervisorConfig struct {
 	// The PCIe Root Port device is used to hot-plug the PCIe device
 	PCIeRootPort uint32
 
+	// ColdPlugVFIO is used to indicate if devices need to be coldplugged on the
+	// root port, switch or no port
+	ColdPlugVFIO hv.PCIePort
+
 	// NumVCPUs specifies default number of vCPUs for the VM.
 	NumVCPUs uint32
 

src/runtime/virtcontainers/kata_agent.go

@@ -1177,7 +1177,8 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, c *Container) []*gr
 		case config.VhostUserBlk:
 			kataDevice = k.appendVhostUserBlkDevice(dev, device, c)
 		case config.DeviceVFIO:
-			kataDevice = k.appendVfioDevice(dev, device, c)
+			k.Logger().Infof("### ColdPlugging container is not adding any VFIO devices")
+			//kataDevice = k.appendVfioDevice(dev, device, c)
 		}
 
 		if kataDevice == nil {

src/runtime/virtcontainers/persist.go

@@ -487,6 +487,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) {
 		DisableNestingChecks:    hconf.DisableNestingChecks,
 		DisableImageNvdimm:      hconf.DisableImageNvdimm,
 		HotplugVFIOOnRootBus:    hconf.HotplugVFIOOnRootBus,
+		ColdPlugVFIO:            hconf.ColdPlugVFIO,
 		PCIeRootPort:            hconf.PCIeRootPort,
 		BootToBeTemplate:        hconf.BootToBeTemplate,
 		BootFromTemplate:        hconf.BootFromTemplate,

src/runtime/virtcontainers/persist/api/config.go

@@ -7,6 +7,7 @@
 package persistapi
 
 import (
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/opencontainers/runc/libcontainer/configs"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
@@ -198,6 +199,10 @@ type HypervisorConfig struct {
 	// root bus instead of a bridge.
 	HotplugVFIOOnRootBus bool
 
+	// ColdPlugVFIO is used to indicate if devices need to be coldlugged on the
+	// root port or a switch or no-port
+	ColdPlugVFIO hv.PCIePort
+
 	// BootToBeTemplate used to indicate if the VM is created to be a template VM
 	BootToBeTemplate bool