runtime: Introduce "vfio_mode" config variable and annotation

In order to support DPDK workloads, we need to change the way VFIO devices
will be handled in Kata containers.  However, the current method, although
it is not remotely OCI compliant has real uses.  Therefore, introduce a new
runtime configuration field "vfio_mode" to control how VFIO devices will be
presented to the container.

We also add a new sandbox annotation -
io.katacontainers.config.runtime.vfio_mode - to override this on a
per-sandbox basis.

For now, the only allowed value is "guest-kernel" which refers to the
current behaviour where VFIO devices added to the container will be bound
to whatever driver in the VM kernel claims them.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>

src/runtime/virtcontainers/device/config/config.go

@@ -187,6 +187,31 @@ type BlockDrive struct {
 	Swap bool
 }
 
+// VFIOMode indicates e behaviour mode for handling devices in the VM
+type VFIOModeType uint32
+
+const (
+	// VFIOModeGuestKernel specifies Kata-specific behaviour
+	// useful in certain cases: VFIO devices specified to Kata are
+	// bound to whatever driver in the VM will take them.  This
+	// requires specialized containers expecting this behaviour to
+	// locate and use the devices
+	VFIOModeGuestKernel = iota
+)
+
+const (
+	vfioModeGuestKernelStr = "guest-kernel"
+)
+
+func (m *VFIOModeType) VFIOSetMode(modeName string) error {
+	switch modeName {
+	case vfioModeGuestKernelStr:
+		*m = VFIOModeGuestKernel
+		return nil
+	}
+	return fmt.Errorf("Unknown VFIO mode %s", modeName)
+}
+
 // VFIODeviceType indicates VFIO device type
 type VFIODeviceType uint32
 

src/runtime/virtcontainers/pkg/annotations/annotations.go

@@ -250,6 +250,10 @@ const (
 
 	// DisableNewNetNs is a sandbox annotation that determines if create a netns for hypervisor process.
 	DisableNewNetNs = kataAnnotRuntimePrefix + "disable_new_netns"
+
+	// VfioMode is a sandbox annotation to specify how attached VFIO devices should be treated
+	// Overrides the runtime.vfio_mode parameter in the global configuration.toml
+	VfioMode = kataAnnotRuntimePrefix + "vfio_mode"
 )
 
 // Agent related annotations

src/runtime/virtcontainers/pkg/oci/utils.go

@@ -116,6 +116,10 @@ type RuntimeConfig struct {
 	//the container network interface
 	InterNetworkModel vc.NetInterworkingModel
 
+	//Determines how VFIO devices should be presented to the
+	//container
+	VfioMode config.VFIOModeType
+
 	Debug bool
 	Trace bool
 
@@ -826,6 +830,13 @@ func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig, r
 		sbConfig.NetworkConfig.InterworkingModel = runtimeConfig.InterNetworkModel
 	}
 
+	if value, ok := ocispec.Annotations[vcAnnotations.VfioMode]; ok {
+		if err := sbConfig.VfioMode.VFIOSetMode(value); err != nil {
+			return fmt.Errorf("Unknown VFIO mode \"%s\" in annotation %s",
+				value, vcAnnotations.VfioMode)
+		}
+	}
+
 	return nil
 }
 
@@ -893,6 +904,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c
 
 		ShmSize: shmSize,
 
+		VfioMode: runtime.VfioMode,
+
 		SystemdCgroup: systemdCgroup,
 
 		SandboxCgroupOnly: runtime.SandboxCgroupOnly,

src/runtime/virtcontainers/sandbox.go

@@ -134,6 +134,8 @@ type SandboxConfig struct {
 
 	ShmSize uint64
 
+	VfioMode config.VFIOModeType
+
 	// SharePidNs sets all containers to share the same sandbox level pid namespace.
 	SharePidNs bool