agent: Bring in VFIO-AP device handling again

This PR is a continuing work for (kata-containers#3679).

This generalizes the previous VFIO device handling which only
focuses on PCI to include AP (IBM Z specific).

Fixes: kata-containers#3678
Signed-off-by: Hyounggyu Choi <Hyounggyu.Choi@ibm.com>

src/agent/src/ap.rs

@@ -1,18 +1,18 @@
-// Copyright (c) IBM Corp. 2022
+// Copyright (c) IBM Corp. 2023
 //
 // SPDX-License-Identifier: Apache-2.0
 //
-
 use std::fmt;
 use std::str::FromStr;
 
 use anyhow::{anyhow, Context};
 
-// IBM Adjunct Processor (AP) is the bus used by IBM Crypto Express hardware security modules on
+// IBM Adjunct Processor (AP) is used for cryptographic operations
-// IBM Z & LinuxONE (s390x)
+// by IBM Crypto Express hardware security modules on IBM zSystem & LinuxONE (s390x).
-// AP bus ID follow the format <xx>.<xxxx> [1, p. 476], where
+// In Linux, virtual cryptographic devices are called AP queues.
-//   - <xx> is the adapter ID, i.e. the card and
+// The name of an AP queue respects a format <xx>.<xxxx> in hexadecimal notation [1, p.467]:
-//   - <xxxx> is the adapter domain.
+//   - <xx> is an adapter ID
+//   - <xxxx> is an adapter domain ID
 // [1] https://www.ibm.com/docs/en/linuxonibm/pdf/lku5dd05.pdf
 
 #[derive(Debug)]
@@ -37,17 +37,17 @@ impl FromStr for Address {
         let split: Vec<&str> = s.split('.').collect();
         if split.len() != 2 {
             return Err(anyhow!(
-                "Wrong AP bus format. It needs to be in the form <xx>.<xxxx>, got {:?}",
+                "Wrong AP bus format. It needs to be in the form <xx>.<xxxx> (e.g. 0a.003f), got {:?}",
                 s
             ));
         }
 
         let adapter_id = u8::from_str_radix(split[0], 16).context(format!(
-            "Wrong AP bus format. AP ID needs to be in the form <xx>, got {:?}",
+            "Wrong AP bus format. AP ID needs to be in the form <xx> (e.g. 0a), got {:?}",
             split[0]
         ))?;
         let adapter_domain = u16::from_str_radix(split[1], 16).context(format!(
-            "Wrong AP bus format. AP domain needs to be in the form <xxxx>, got {:?}",
+            "Wrong AP bus format. AP domain needs to be in the form <xxxx> (e.g. 003f), got {:?}",
             split[1]
         ))?;
 

src/agent/src/device.rs

@@ -764,8 +764,8 @@ async fn vfio_pci_device_handler(
     let mut group = None;
 
     for opt in device.options.iter() {
-        let (host, pcipath) =
+        let (host, pcipath) = split_vfio_pci_option(opt)
-            split_vfio_pci_option(opt).ok_or_else(|| anyhow!("Malformed VFIO option {:?}", opt))?;
+            .ok_or_else(|| anyhow!("Malformed VFIO PCI option {:?}", opt))?;
         let host =
             pci::Address::from_str(host).context("Bad host PCI address in VFIO option {:?}")?;
         let pcipath = pci::Path::from_str(pcipath)?;
@@ -809,7 +809,7 @@ async fn vfio_pci_device_handler(
 
 // The VFIO AP (Adjunct Processor) device handler takes all the APQNs provided as device options
 // and awaits them. It sets the minimum AP rescan time of 5 seconds and temporarily adds that
-// amoutn to the hotplug timeout.
+// amount to the hotplug timeout.
 #[cfg(target_arch = "s390x")]
 #[instrument]
 async fn vfio_ap_device_handler(

src/runtime/pkg/device/drivers/vfio.go

@@ -118,6 +118,8 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece
 				Type:      config.VFIOAPDeviceMediatedType,
 				APDevices: devices,
 			}
+		default:
+			return fmt.Errorf("Failed to append device: VFIO device type unrecognized")
 		}
 		device.VfioDevs = append(device.VfioDevs, &vfio)
 	}
@@ -216,12 +218,9 @@ func (device *VFIODevice) Load(ds config.DeviceState) {
 	for _, dev := range ds.VFIODevs {
 		var vfio config.VFIODev
 
-		if (*device.VfioDevs[0]).GetType() == config.VFIOAPDeviceMediatedType {
+		vfioDeviceType := (*device.VfioDevs[0]).GetType()
-			vfio = config.VFIOAPDev{
+		switch vfioDeviceType {
-				ID:       *(*dev).GetID(),
+		case config.VFIOPCIDeviceNormalType, config.VFIOPCIDeviceMediatedType:
-				SysfsDev: *(*dev).GetSysfsDev(),
-			}
-		} else {
 			bdf := ""
 			if pciDev, ok := (*dev).(config.VFIOPCIDev); ok {
 				bdf = pciDev.BDF
@@ -232,6 +231,16 @@ func (device *VFIODevice) Load(ds config.DeviceState) {
 				BDF:      bdf,
 				SysfsDev: *(*dev).GetSysfsDev(),
 			}
+		case config.VFIOAPDeviceMediatedType:
+			vfio = config.VFIOAPDev{
+				ID:       *(*dev).GetID(),
+				SysfsDev: *(*dev).GetSysfsDev(),
+			}
+		default:
+			deviceLogger().WithError(
+				fmt.Errorf("VFIO device type unrecognized"),
+			).Error("Failed to append device")
+			return
 		}
 
 		device.VfioDevs = append(device.VfioDevs, &vfio)

src/runtime/virtcontainers/clh.go

@@ -856,13 +856,8 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error {
 	ctx, cancel := context.WithTimeout(context.Background(), clhHotPlugAPITimeout*time.Second)
 	defer cancel()
 
-	pciDevice, ok := (*device).(config.VFIOPCIDev)
-	if !ok {
-		return fmt.Errorf("VFIO device %+v is not PCI, only PCI is supported in Cloud Hypervisor", device)
-	}
-
 	// Create the clh device config via the constructor to ensure default values are properly assigned
-	clhDevice := *chclient.NewDeviceConfig(device.SysfsDev)
+	clhDevice := *chclient.NewDeviceConfig(*(*device).GetSysfsDev())
 	pciInfo, _, err := cl.VmAddDevicePut(ctx, clhDevice)
 	if err != nil {
 		return fmt.Errorf("Failed to hotplug device %+v %s", device, openAPIClientError(err))
@@ -885,6 +880,11 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error {
 	}
 
 	guestPciPath, err := types.PciPathFromString(tokens[0])
+
+	pciDevice, ok := (*device).(config.VFIOPCIDev)
+	if !ok {
+		return fmt.Errorf("VFIO device %+v is not PCI, only PCI is supported in Cloud Hypervisor", device)
+	}
 	pciDevice.GuestPciPath = guestPciPath
 	*device = pciDevice
 

src/runtime/virtcontainers/utils/utils_linux.go

@@ -141,18 +141,6 @@ func GetDevicePathAndFsTypeOptions(mountPoint string) (devicePath, fsType string
 	}
 }
 
-// IsAPVFIOMediatedDevice decides whether a device is a VFIO-AP device
-// by checking for the existence of "vfio_ap" in the path
-func IsAPVFIOMediatedDevice(sysfsdev string) bool {
-	split := strings.Split(sysfsdev, string(os.PathSeparator))
-	for _, el := range split {
-		if el == vfioAPSysfsDir {
-			return true
-		}
-	}
-	return false
-}
-
 func waitProcessUsingPidfd(pid int, timeoutSecs uint, logger *logrus.Entry) (bool, error) {
 	pidfd, err := unix.PidfdOpen(pid, 0)