From f666f8e2df6bfee78c34d1bf449865bd9c1f4b8f Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 8 Mar 2022 18:51:08 +0100 Subject: [PATCH] agent: Add VFIO-AP device handling Initial VFIO-AP support (#578) was simple, but somewhat hacky; a different code path would be chosen for performing the hotplug, and agent-side device handling was bound to knowing the assigned queue numbers (APQNs) through some other means; plus the code for awaiting them was written for the Go agent and never released. This code also artificially increased the hotplug timeout to wait for the (relatively expensive, thus limited to 5 seconds at the quickest) AP rescan, which is impractical for e.g. common k8s timeouts. Since then, the general handling logic was improved (#1190), but it assumed PCI in several places. In the runtime, introduce and parse AP devices. Annotate them as such when passing to the agent, and include information about the associated APQNs. The agent awaits the passed APQNs through uevents and triggers a rescan directly. Fixes: #3678 Signed-off-by: Jakob Naucke --- src/agent/src/ap.rs | 79 ++++++++++++++++++ src/agent/src/device.rs | 89 +++++++++++++++++++++ src/agent/src/linux_abi.rs | 2 + src/agent/src/main.rs | 1 + src/runtime/pkg/device/config/config.go | 29 +++++++ src/runtime/pkg/device/drivers/utils.go | 49 +++++++++--- src/runtime/pkg/device/drivers/vfio.go | 67 ++++++++++------ src/runtime/pkg/device/drivers/vfio_test.go | 2 +- src/runtime/virtcontainers/clh.go | 4 +- src/runtime/virtcontainers/kata_agent.go | 83 ++++++++++--------- src/runtime/virtcontainers/qemu.go | 4 + 11 files changed, 335 insertions(+), 74 deletions(-) create mode 100644 src/agent/src/ap.rs diff --git a/src/agent/src/ap.rs b/src/agent/src/ap.rs new file mode 100644 index 000000000..454ada9ff --- /dev/null +++ b/src/agent/src/ap.rs @@ -0,0 +1,79 @@ +// Copyright (c) IBM Corp. 2022 +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::fmt; +use std::str::FromStr; + +use anyhow::{anyhow, Context}; + +// IBM Adjunct Processor (AP) is the bus used by IBM Crypto Express hardware security modules on +// IBM Z & LinuxONE (s390x) +// AP bus ID follow the format . [1, p. 476], where +// - is the adapter ID, i.e. the card and +// - is the adapter domain. +// [1] https://www.ibm.com/docs/en/linuxonibm/pdf/lku5dd05.pdf + +#[derive(Debug)] +pub struct Address { + pub adapter_id: u8, + pub adapter_domain: u16, +} + +impl Address { + pub fn new(adapter_id: u8, adapter_domain: u16) -> Address { + Address { + adapter_id, + adapter_domain, + } + } +} + +impl FromStr for Address { + type Err = anyhow::Error; + + fn from_str(s: &str) -> anyhow::Result { + let split: Vec<&str> = s.split('.').collect(); + if split.len() != 2 { + return Err(anyhow!( + "Wrong AP bus format. It needs to be in the form ., got {:?}", + s + )); + } + + let adapter_id = u8::from_str_radix(split[0], 16).context(format!( + "Wrong AP bus format. AP ID needs to be in the form , got {:?}", + split[0] + ))?; + let adapter_domain = u16::from_str_radix(split[1], 16).context(format!( + "Wrong AP bus format. AP domain needs to be in the form , got {:?}", + split[1] + ))?; + + Ok(Address::new(adapter_id, adapter_domain)) + } +} + +impl fmt::Display for Address { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!(f, "{:02x}.{:04x}", self.adapter_id, self.adapter_domain) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_from_str() { + let device = Address::from_str("a.1").unwrap(); + assert_eq!(format!("{}", device), "0a.0001"); + + assert!(Address::from_str("").is_err()); + assert!(Address::from_str(".").is_err()); + assert!(Address::from_str("0.0.0").is_err()); + assert!(Address::from_str("0g.0000").is_err()); + assert!(Address::from_str("0a.10000").is_err()); + } +} diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 23bc154a7..b95e304fb 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -50,11 +50,13 @@ pub const DRIVER_VFIO_PCI_GK_TYPE: &str = "vfio-pci-gk"; // VFIO PCI device to be bound to vfio-pci and made available inside the // container as a VFIO device node pub const DRIVER_VFIO_PCI_TYPE: &str = "vfio-pci"; +pub const DRIVER_VFIO_AP_TYPE: &str = "vfio-ap"; pub const DRIVER_OVERLAYFS_TYPE: &str = "overlayfs"; pub const FS_TYPE_HUGETLB: &str = "hugetlbfs"; cfg_if! { if #[cfg(target_arch = "s390x")] { + use crate::ap; use crate::ccw; } } @@ -406,6 +408,39 @@ async fn get_vfio_device_name(sandbox: &Arc>, grp: IommuGroup) -> Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname)) } +#[cfg(target_arch = "s390x")] +#[derive(Debug)] +struct ApMatcher { + syspath: String, +} + +#[cfg(target_arch = "s390x")] +impl ApMatcher { + fn new(address: ap::Address) -> ApMatcher { + ApMatcher { + syspath: format!( + "{}/card{:02x}/{}", + AP_ROOT_BUS_PATH, address.adapter_id, address + ), + } + } +} + +#[cfg(target_arch = "s390x")] +impl UeventMatcher for ApMatcher { + fn is_match(&self, uev: &Uevent) -> bool { + uev.action == "add" && uev.devpath == self.syspath + } +} + +#[cfg(target_arch = "s390x")] +#[instrument] +async fn wait_for_ap_device(sandbox: &Arc>, address: ap::Address) -> Result<()> { + let matcher = ApMatcher::new(address); + wait_for_uevent(sandbox, matcher).await?; + Ok(()) +} + /// Scan SCSI bus for the given SCSI address(SCSI-Id and LUN) #[instrument] fn scan_scsi_bus(scsi_addr: &str) -> Result<()> { @@ -772,6 +807,28 @@ async fn vfio_pci_device_handler( }) } +// The VFIO AP (Adjunct Processor) device handler takes all the APQNs provided as device options +// and awaits them. It sets the minimum AP rescan time of 5 seconds and temporarily adds that +// amoutn to the hotplug timeout. +#[cfg(target_arch = "s390x")] +#[instrument] +async fn vfio_ap_device_handler( + device: &Device, + sandbox: &Arc>, +) -> Result { + // Force AP bus rescan + fs::write(AP_SCANS_PATH, "1")?; + for apqn in device.options.iter() { + wait_for_ap_device(sandbox, ap::Address::from_str(apqn)?).await?; + } + Ok(Default::default()) +} + +#[cfg(not(target_arch = "s390x"))] +async fn vfio_ap_device_handler(_: &Device, _: &Arc>) -> Result { + Err(anyhow!("AP is only supported on s390x")) +} + #[instrument] pub async fn add_devices( devices: &[Device], @@ -840,6 +897,7 @@ async fn add_device(device: &Device, sandbox: &Arc>) -> Result { vfio_pci_device_handler(device, sandbox).await } + DRIVER_VFIO_AP_TYPE => vfio_ap_device_handler(device, sandbox).await, _ => Err(anyhow!("Unknown device type {}", device.field_type)), } } @@ -1583,4 +1641,35 @@ mod tests { // Test dev2 assert!(pci_iommu_group(&syspci, dev2).is_err()); } + + #[cfg(target_arch = "s390x")] + #[tokio::test] + async fn test_vfio_ap_matcher() { + let subsystem = "ap"; + let card = "0a"; + let relpath = format!("{}.0001", card); + + let mut uev = Uevent::default(); + uev.action = U_EVENT_ACTION_ADD.to_string(); + uev.subsystem = subsystem.to_string(); + uev.devpath = format!("{}/card{}/{}", AP_ROOT_BUS_PATH, card, relpath); + + let ap_address = ap::Address::from_str(&relpath).unwrap(); + let matcher = ApMatcher::new(ap_address); + + assert!(matcher.is_match(&uev)); + + let mut uev_remove = uev.clone(); + uev_remove.action = U_EVENT_ACTION_REMOVE.to_string(); + assert!(!matcher.is_match(&uev_remove)); + + let mut uev_other_device = uev.clone(); + uev_other_device.devpath = format!( + "{}/card{}/{}", + AP_ROOT_BUS_PATH, + card, + format!("{}.0002", card) + ); + assert!(!matcher.is_match(&uev_other_device)); + } } diff --git a/src/agent/src/linux_abi.rs b/src/agent/src/linux_abi.rs index 9df398d61..042acd0ae 100644 --- a/src/agent/src/linux_abi.rs +++ b/src/agent/src/linux_abi.rs @@ -69,6 +69,8 @@ pub fn create_pci_root_bus_path() -> String { cfg_if! { if #[cfg(target_arch = "s390x")] { pub const CCW_ROOT_BUS_PATH: &str = "/devices/css0"; + pub const AP_ROOT_BUS_PATH: &str = "/devices/ap"; + pub const AP_SCANS_PATH: &str = "/sys/bus/ap/scans"; } } diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 4ca89a9a1..085ea396d 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -75,6 +75,7 @@ mod tracer; cfg_if! { if #[cfg(target_arch = "s390x")] { + mod ap; mod ccw; } } diff --git a/src/runtime/pkg/device/config/config.go b/src/runtime/pkg/device/config/config.go index 6bcbed32c..dee6291ed 100644 --- a/src/runtime/pkg/device/config/config.go +++ b/src/runtime/pkg/device/config/config.go @@ -263,6 +263,9 @@ const ( // VFIOPCIDeviceMediatedType is a VFIO PCI mediated device type VFIOPCIDeviceMediatedType + + // VFIOAPDeviceMediatedType is a VFIO AP mediated device type + VFIOAPDeviceMediatedType ) type VFIODev interface { @@ -316,6 +319,32 @@ func (d VFIOPCIDev) GetSysfsDev() *string { return &d.SysfsDev } +type VFIOAPDev struct { + // ID is used to identify this drive in the hypervisor options. + ID string + + // sysfsdev of VFIO mediated device + SysfsDev string + + // APDevices are the Adjunct Processor devices assigned to the mdev + APDevices []string + + // Type of VFIO device + Type VFIODeviceType +} + +func (d VFIOAPDev) GetID() *string { + return &d.ID +} + +func (d VFIOAPDev) GetType() VFIODeviceType { + return d.Type +} + +func (d VFIOAPDev) GetSysfsDev() *string { + return &d.SysfsDev +} + // RNGDev represents a random number generator device type RNGDev struct { // ID is used to identify the device in the hypervisor options. diff --git a/src/runtime/pkg/device/drivers/utils.go b/src/runtime/pkg/device/drivers/utils.go index abdf5d816..79e00adbd 100644 --- a/src/runtime/pkg/device/drivers/utils.go +++ b/src/runtime/pkg/device/drivers/utils.go @@ -89,18 +89,47 @@ func readPCIProperty(propertyPath string) (string, error) { return strings.Split(string(buf), "\n")[0], nil } -func GetVFIODeviceType(deviceFileName string) config.VFIODeviceType { +func GetVFIODeviceType(deviceFilePath string) (config.VFIODeviceType, error) { + deviceFileName := filepath.Base(deviceFilePath) + //For example, 0000:04:00.0 tokens := strings.Split(deviceFileName, ":") - vfioDeviceType := config.VFIODeviceErrorType if len(tokens) == 3 { - vfioDeviceType = config.VFIOPCIDeviceNormalType - } else { - //For example, 83b8f4f2-509f-382f-3c1e-e6bfe0fa1001 - tokens = strings.Split(deviceFileName, "-") - if len(tokens) == 5 { - vfioDeviceType = config.VFIOPCIDeviceMediatedType - } + return config.VFIOPCIDeviceNormalType, nil } - return vfioDeviceType + + //For example, 83b8f4f2-509f-382f-3c1e-e6bfe0fa1001 + tokens = strings.Split(deviceFileName, "-") + if len(tokens) != 5 { + return config.VFIODeviceErrorType, fmt.Errorf("Incorrect tokens found while parsing VFIO details: %s", deviceFileName) + } + + deviceSysfsDev, err := GetSysfsDev(deviceFilePath) + if err != nil { + return config.VFIODeviceErrorType, err + } + + if strings.HasPrefix(deviceSysfsDev, vfioAPSysfsDir) { + return config.VFIOAPDeviceMediatedType, nil + } + + return config.VFIOPCIDeviceMediatedType, nil +} + +// GetSysfsDev returns the sysfsdev of mediated device +// Expected input string format is absolute path to the sysfs dev node +// eg. /sys/kernel/iommu_groups/0/devices/f79944e4-5a3d-11e8-99ce-479cbab002e4 +func GetSysfsDev(sysfsDevStr string) (string, error) { + return filepath.EvalSymlinks(sysfsDevStr) +} + +// GetAPVFIODevices retrieves all APQNs associated with a mediated VFIO-AP +// device +func GetAPVFIODevices(sysfsdev string) ([]string, error) { + data, err := os.ReadFile(filepath.Join(sysfsdev, "matrix")) + if err != nil { + return []string{}, err + } + // Split by newlines, omitting final newline + return strings.Split(string(data[:len(data)-1]), "\n"), nil } diff --git a/src/runtime/pkg/device/drivers/vfio.go b/src/runtime/pkg/device/drivers/vfio.go index a07a7df73..3450e61d9 100644 --- a/src/runtime/pkg/device/drivers/vfio.go +++ b/src/runtime/pkg/device/drivers/vfio.go @@ -30,6 +30,7 @@ const ( iommuGroupPath = "/sys/bus/pci/devices/%s/iommu_group" vfioDevPath = "/dev/vfio/%s" pcieRootPortPrefix = "rp" + vfioAPSysfsDir = "/sys/devices/vfio_ap" ) var ( @@ -106,6 +107,17 @@ func (device *VFIODevice) Attach(ctx context.Context, devReceiver api.DeviceRece AllPCIeDevs[deviceBDF] = true } vfio = vfioPCI + case config.VFIOAPDeviceMediatedType: + devices, err := GetAPVFIODevices(deviceSysfsDev) + if err != nil { + return err + } + vfio = config.VFIOAPDev{ + ID: id, + SysfsDev: deviceSysfsDev, + Type: config.VFIOAPDeviceMediatedType, + APDevices: devices, + } } device.VfioDevs = append(device.VfioDevs, &vfio) } @@ -190,16 +202,7 @@ func (device *VFIODevice) Save() config.DeviceState { devs := device.VfioDevs for _, dev := range devs { if dev != nil { - bdf := "" - if pciDev, ok := (*dev).(config.VFIOPCIDev); ok { - bdf = pciDev.BDF - } - ds.VFIODevs = append(ds.VFIODevs, &persistapi.VFIODev{ - ID: *(*dev).GetID(), - Type: uint32((*dev).GetType()), - BDF: bdf, - SysfsDev: *(*dev).GetSysfsDev(), - }) + ds.VFIODevs = append(ds.VFIODevs, dev) } } return ds @@ -211,20 +214,38 @@ func (device *VFIODevice) Load(ds config.DeviceState) { device.GenericDevice.Load(ds) for _, dev := range ds.VFIODevs { - var vfioDev config.VFIODev = config.VFIOPCIDev{ - ID: dev.ID, - Type: config.VFIODeviceType(dev.Type), - BDF: dev.BDF, - SysfsDev: dev.SysfsDev, + var vfio config.VFIODev + + if (*device.VfioDevs[0]).GetType() == config.VFIOAPDeviceMediatedType { + vfio = config.VFIOAPDev{ + ID: *(*dev).GetID(), + SysfsDev: *(*dev).GetSysfsDev(), + } + } else { + bdf := "" + if pciDev, ok := (*dev).(config.VFIOPCIDev); ok { + bdf = pciDev.BDF + } + vfio = config.VFIOPCIDev{ + ID: *(*dev).GetID(), + Type: config.VFIODeviceType((*dev).GetType()), + BDF: bdf, + SysfsDev: *(*dev).GetSysfsDev(), + } } - device.VfioDevs = append(device.VfioDevs, &vfioDev) + + device.VfioDevs = append(device.VfioDevs, &vfio) } } // It should implement GetAttachCount() and DeviceID() as api.Device implementation // here it shares function from *GenericDevice so we don't need duplicate codes func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceSysfsDev string, vfioDeviceType config.VFIODeviceType, err error) { - vfioDeviceType = GetVFIODeviceType(deviceFileName) + sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName) + vfioDeviceType, err = GetVFIODeviceType(sysfsDevStr) + if err != nil { + return deviceBDF, deviceSysfsDev, vfioDeviceType, err + } switch vfioDeviceType { case config.VFIOPCIDeviceNormalType: @@ -235,8 +256,11 @@ func getVFIODetails(deviceFileName, iommuDevicesPath string) (deviceBDF, deviceS case config.VFIOPCIDeviceMediatedType: // Get sysfsdev of device eg. /sys/devices/pci0000:00/0000:00:02.0/f79944e4-5a3d-11e8-99ce-479cbab002e4 sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName) - deviceSysfsDev, err = getSysfsDev(sysfsDevStr) + deviceSysfsDev, err = GetSysfsDev(sysfsDevStr) deviceBDF = getBDF(getMediatedBDF(deviceSysfsDev)) + case config.VFIOAPDeviceMediatedType: + sysfsDevStr := filepath.Join(iommuDevicesPath, deviceFileName) + deviceSysfsDev, err = GetSysfsDev(sysfsDevStr) default: err = fmt.Errorf("Incorrect tokens found while parsing vfio details: %s", deviceFileName) } @@ -264,13 +288,6 @@ func getBDF(deviceSysStr string) string { return tokens[1] } -// getSysfsDev returns the sysfsdev of mediated device -// Expected input string format is absolute path to the sysfs dev node -// eg. /sys/kernel/iommu_groups/0/devices/f79944e4-5a3d-11e8-99ce-479cbab002e4 -func getSysfsDev(sysfsDevStr string) (string, error) { - return filepath.EvalSymlinks(sysfsDevStr) -} - // BindDevicetoVFIO binds the device to vfio driver after unbinding from host. // Will be called by a network interface or a generic pcie device. func BindDevicetoVFIO(bdf, hostDriver, vendorDeviceID string) (string, error) { diff --git a/src/runtime/pkg/device/drivers/vfio_test.go b/src/runtime/pkg/device/drivers/vfio_test.go index 8ee008e3b..6a1ab61eb 100644 --- a/src/runtime/pkg/device/drivers/vfio_test.go +++ b/src/runtime/pkg/device/drivers/vfio_test.go @@ -34,7 +34,7 @@ func TestGetVFIODetails(t *testing.T) { switch vfioDeviceType { case config.VFIOPCIDeviceNormalType: assert.Equal(t, d.expectedStr, deviceBDF) - case config.VFIOPCIDeviceMediatedType: + case config.VFIOPCIDeviceMediatedType, config.VFIOAPDeviceMediatedType: assert.Equal(t, d.expectedStr, deviceSysfsDev) default: assert.NotNil(t, err) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index d9be15082..8799d6a93 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -867,7 +867,7 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error { if err != nil { return fmt.Errorf("Failed to hotplug device %+v %s", device, openAPIClientError(err)) } - clh.devicesIds[device.ID] = pciInfo.GetId() + clh.devicesIds[*(*device).GetID()] = pciInfo.GetId() // clh doesn't use bridges, so the PCI path is simply the slot // number of the device. This will break if clh starts using @@ -884,7 +884,7 @@ func (clh *cloudHypervisor) hotPlugVFIODevice(device *config.VFIODev) error { return fmt.Errorf("Unexpected PCI address %q from clh hotplug", pciInfo.Bdf) } - guestPciPath, err := vcTypes.PciPathFromString(tokens[0]) + guestPciPath, err := types.PciPathFromString(tokens[0]) pciDevice.GuestPciPath = guestPciPath *device = pciDevice diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 60010a14a..9e5c8b34f 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -77,38 +77,39 @@ const ( ) var ( - checkRequestTimeout = 30 * time.Second - defaultRequestTimeout = 60 * time.Second - errorMissingOCISpec = errors.New("Missing OCI specification") - defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/" - defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/" - defaultKataGuestNydusRootDir = "/run/kata-containers/shared/" - mountGuestTag = "kataShared" - defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/" - type9pFs = "9p" - typeVirtioFS = "virtiofs" - typeOverlayFS = "overlay" - kata9pDevType = "9p" - kataMmioBlkDevType = "mmioblk" - kataBlkDevType = "blk" - kataBlkCCWDevType = "blk-ccw" - kataSCSIDevType = "scsi" - kataNvdimmDevType = "nvdimm" - kataVirtioFSDevType = "virtio-fs" - kataOverlayDevType = "overlayfs" - kataWatchableBindDevType = "watchable-bind" - kataVfioDevType = "vfio" // VFIO device to used as VFIO in the container - kataVfioGuestKernelDevType = "vfio-gk" // VFIO device for consumption by the guest kernel - sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L,cache=mmap", "nodev"} - sharedDirVirtioFSOptions = []string{} - sharedDirVirtioFSDaxOptions = "dax" - shmDir = "shm" - kataEphemeralDevType = "ephemeral" - defaultEphemeralPath = filepath.Join(defaultKataGuestSandboxDir, kataEphemeralDevType) - grpcMaxDataSize = int64(1024 * 1024) - localDirOptions = []string{"mode=0777"} - maxHostnameLen = 64 - GuestDNSFile = "/etc/resolv.conf" + checkRequestTimeout = 30 * time.Second + defaultRequestTimeout = 60 * time.Second + errorMissingOCISpec = errors.New("Missing OCI specification") + defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/" + defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/" + defaultKataGuestNydusRootDir = "/run/kata-containers/shared/" + mountGuestTag = "kataShared" + defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/" + type9pFs = "9p" + typeVirtioFS = "virtiofs" + typeOverlayFS = "overlay" + kata9pDevType = "9p" + kataMmioBlkDevType = "mmioblk" + kataBlkDevType = "blk" + kataBlkCCWDevType = "blk-ccw" + kataSCSIDevType = "scsi" + kataNvdimmDevType = "nvdimm" + kataVirtioFSDevType = "virtio-fs" + kataOverlayDevType = "overlayfs" + kataWatchableBindDevType = "watchable-bind" + kataVfioPciDevType = "vfio-pci" // VFIO PCI device to used as VFIO in the container + kataVfioPciGuestKernelDevType = "vfio-pci-gk" // VFIO PCI device for consumption by the guest kernel + kataVfioApDevType = "vfio-ap" + sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L,cache=mmap", "nodev"} + sharedDirVirtioFSOptions = []string{} + sharedDirVirtioFSDaxOptions = "dax" + shmDir = "shm" + kataEphemeralDevType = "ephemeral" + defaultEphemeralPath = filepath.Join(defaultKataGuestSandboxDir, kataEphemeralDevType) + grpcMaxDataSize = int64(1024 * 1024) + localDirOptions = []string{"mode=0777"} + maxHostnameLen = 64 + GuestDNSFile = "/etc/resolv.conf" ) const ( @@ -1126,6 +1127,11 @@ func (k *kataAgent) appendVfioDevice(dev ContainerDevice, device api.Device, c * // DDDD:BB:DD.F is the device's PCI address on the // *host*. is the device's PCI path in the guest // (see qomGetPciPath() for details). + // + // For VFIO-AP, one VFIO group could include several queue devices. They are + // identified by APQNs (Adjunct Processor Queue Numbers), which do not differ + // between host and guest. They are passed as options so they can be awaited + // by the agent. kataDevice := &grpc.Device{ ContainerPath: dev.ContainerPath, Type: kataVfioPciDevType, @@ -1141,10 +1147,15 @@ func (k *kataAgent) appendVfioDevice(dev ContainerDevice, device api.Device, c * kataDevice.Type = kataVfioPciGuestKernelDevType } - kataDevice.Options = make([]string, len(devList)) - for i, device := range devList { - pciDevice := (*device).(config.VFIOPCIDev) - kataDevice.Options[i] = fmt.Sprintf("0000:%s=%s", pciDevice.BDF, pciDevice.GuestPciPath) + if (*devList[0]).GetType() == config.VFIOAPDeviceMediatedType { + kataDevice.Type = kataVfioApDevType + kataDevice.Options = (*devList[0]).(config.VFIOAPDev).APDevices + } else { + kataDevice.Options = make([]string, len(devList)) + for i, device := range devList { + pciDevice := (*device).(config.VFIOPCIDev) + kataDevice.Options[i] = fmt.Sprintf("0000:%s=%s", pciDevice.BDF, pciDevice.GuestPciPath) + } } return kataDevice diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 6354b9d8a..16872c126 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -1753,6 +1753,8 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op } else { err = q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, *(*device).GetSysfsDev(), "", pciDevice.Bus, romFile) } + case config.VFIOAPDeviceMediatedType: + err = q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, *(*device).GetSysfsDev()) } } else { addr, bridge, err := q.arch.addDeviceToBridge(ctx, devID, types.PCI) @@ -1775,6 +1777,8 @@ func (q *qemu) hotplugVFIODevice(ctx context.Context, device *config.VFIODev, op err = q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, pciDevice.BDF, addr, bridge.ID, romFile) case config.VFIOPCIDeviceMediatedType: err = q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, *(*device).GetSysfsDev(), addr, bridge.ID, romFile) + case config.VFIOAPDeviceMediatedType: + err = q.qmpMonitorCh.qmp.ExecuteAPVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, *(*device).GetSysfsDev()) default: return fmt.Errorf("Incorrect VFIO device type found") }