gpu: Check for VFIO port assignments

Bailing out early if the port is wrong, allowed port settings are
no-port, root-port, switch-port

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>

src/runtime/pkg/containerd-shim-v2/create_test.go

@@ -20,6 +20,7 @@ import (
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"
 
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
@@ -308,6 +309,7 @@ func TestCreateContainerConfigFail(t *testing.T) {
 }
 
 func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err error) {
+	var coldPlugVFIO hv.PCIePort
 	if dir == "" {
 		return "", fmt.Errorf("BUG: need directory")
 	}
@@ -332,6 +334,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err
 	disableNewNetNs := false
 	sharedFS := "virtio-9p"
 	virtioFSdaemon := path.Join(dir, "virtiofsd")
+	coldPlugVFIO = hv.RootPort
 
 	configFileOptions := ktu.RuntimeConfigOptions{
 		Hypervisor:           "qemu",
@@ -350,6 +353,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err
 		DisableNewNetNs:      disableNewNetNs,
 		SharedFS:             sharedFS,
 		VirtioFSDaemon:       virtioFSdaemon,
+		ColdPlugVFIO:         coldPlugVFIO,
 	}
 
 	runtimeConfigFileData := ktu.MakeRuntimeConfigFileData(configFileOptions)

src/runtime/pkg/hypervisors/hypervisor_state.go

@@ -48,10 +48,12 @@ func (p PCIePort) String() string {
 		return "root-port"
 	case SwitchPort:
 		return "switch-port"
+	case BridgePort:
+		return "bridge-port"
 	case NoPort:
 		return "no-port"
 	}
-	return fmt.Sprintf("unknown PCIePort: %s", string(p))
+	return fmt.Sprintf("<unknown PCIePort: %s>", string(p))
 }
 
 type HypervisorState struct {

src/runtime/pkg/katautils/config-settings.go.in

@@ -9,6 +9,10 @@
 
 package katautils
 
+import (
+	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
+)
+
 // name is the name of the runtime
 var NAME = "@RUNTIME_NAME@"
 
@@ -104,4 +108,4 @@ const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock"
 // Default config file used by stateless systems.
 var defaultRuntimeConfiguration = "@CONFIG_PATH@"
 
-const defaultColdPlugVFIO = "no-port"
+const defaultColdPlugVFIO = hv.NoPort

src/runtime/pkg/katautils/config.go

@@ -287,6 +287,13 @@ func (h hypervisor) firmware() (string, error) {
 	return ResolvePath(p)
 }
 
+func (h hypervisor) coldPlugVFIO() hv.PCIePort {
+	if h.ColdPlugVFIO == "" {
+		return defaultColdPlugVFIO
+	}
+	return h.ColdPlugVFIO
+}
+
 func (h hypervisor) firmwareVolume() (string, error) {
 	p := h.FirmwareVolume
 
@@ -856,7 +863,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		Msize9p:                 h.msize9p(),
 		DisableImageNvdimm:      h.DisableImageNvdimm,
 		HotplugVFIOOnRootBus:    h.HotplugVFIOOnRootBus,
-		ColdPlugVFIO:            h.ColdPlugVFIO,
+		ColdPlugVFIO:            h.coldPlugVFIO(),
 		PCIeRootPort:            h.PCIeRootPort,
 		DisableVhostNet:         h.DisableVhostNet,
 		EnableVhostUserStore:    h.EnableVhostUserStore,
@@ -1051,7 +1058,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		EnableIOThreads:                h.EnableIOThreads,
 		Msize9p:                        h.msize9p(),
 		HotplugVFIOOnRootBus:           h.HotplugVFIOOnRootBus,
-		ColdPlugVFIO:                   h.ColdPlugVFIO,
+		ColdPlugVFIO:                   h.coldPlugVFIO(),
 		PCIeRootPort:                   h.PCIeRootPort,
 		DisableVhostNet:                true,
 		GuestHookPath:                  h.guestHookPath(),
@@ -1655,9 +1662,32 @@ func checkConfig(config oci.RuntimeConfig) error {
 		return err
 	}
 
+	coldPlugVFIO := config.HypervisorConfig.ColdPlugVFIO
+	machineType := config.HypervisorConfig.HypervisorMachineType
+	if err := checkPCIeConfig(coldPlugVFIO, machineType); err != nil {
+		return err
+	}
+
 	return nil
 }
 
+// checkPCIeConfig ensures the PCIe configuration is valid.
+// Only allow one of the following settings for cold-plug:
+// no-port, root-port, switch-port
+func checkPCIeConfig(vfioPort hv.PCIePort, machineType string) error {
+	// Currently only QEMU q35 supports advanced PCIe topologies
+	// firecracker, dragonball do not have right now any PCIe support
+	if machineType != "q35" {
+		return nil
+	}
+	if vfioPort == hv.NoPort || vfioPort == hv.RootPort || vfioPort == hv.SwitchPort {
+		return nil
+	}
+
+	return fmt.Errorf("invalid vfio_port=%s setting, allowed values %s, %s, %s",
+		vfioPort, hv.NoPort, hv.RootPort, hv.SwitchPort)
+}
+
 // checkNetNsConfig performs sanity checks on disable_new_netns config.
 // Because it is an expert option and conflicts with some other common configs.
 func checkNetNsConfig(config oci.RuntimeConfig) error {

src/runtime/virtcontainers/sandbox.go

@@ -623,7 +623,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 
 	// If we have a confidential guest we need to cold-plug the PCIe VFIO devices
 	// until we have TDISP/IDE PCIe support.
-	coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO == hv.RootPort)
+	coldPlugVFIO := (sandboxConfig.HypervisorConfig.ColdPlugVFIO != hv.NoPort)
 	var devs []config.DeviceInfo
 	for cnt, containers := range sandboxConfig.Containers {
 		for dev, device := range containers.DeviceInfos {