From 8758ce26b73b2ec7f91ec04a2d313100173c9cfe Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Mon, 28 Jun 2021 16:06:37 +0200 Subject: [PATCH] agent: Enable virtio-blk-ccw Forward-port of https://github.com/kata-containers/agent/pull/600. Enable virtio-blk-ccw devices in agent (virtio-blk for s390x, already enabled in runtime). Fixes: #2026 Signed-off-by: Jakob Naucke --- src/agent/src/ccw.rs | 140 +++++++++++++++++++++++++++++++++++++ src/agent/src/device.rs | 135 ++++++++++++++++++++++++++++++++++- src/agent/src/linux_abi.rs | 4 ++ src/agent/src/main.rs | 2 + src/agent/src/mount.rs | 31 ++++++++ src/agent/src/uevent.rs | 2 +- 6 files changed, 312 insertions(+), 2 deletions(-) create mode 100644 src/agent/src/ccw.rs diff --git a/src/agent/src/ccw.rs b/src/agent/src/ccw.rs new file mode 100644 index 000000000..f5cb12151 --- /dev/null +++ b/src/agent/src/ccw.rs @@ -0,0 +1,140 @@ +// Copyright (c) IBM Corp. 2021 +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::fmt; +use std::str::FromStr; + +use anyhow::anyhow; + +// CCW bus ID follow the format .. [1, p. 11], where +// - is the channel subsystem ID, which is always 0 from the guest side, but different from +// the host side, e.g. 0xfe for virtio-*-ccw [1, p. 435], +// - is the subchannel set ID, which ranges from 0-3 [2], and +// - is the device number (0000-ffff; leading zeroes can be omitted, +// e.g. 3 instead of 0003). +// [1] https://www.ibm.com/docs/en/linuxonibm/pdf/lku4dd04.pdf +// [2] https://qemu.readthedocs.io/en/latest/system/s390x/css.html + +// Maximum subchannel set ID +const SUBCHANNEL_SET_MAX: u8 = 3; + +// CCW device. From the guest side, the first field is always 0 and can therefore be omitted. +#[derive(Copy, Clone, Debug)] +pub struct Device { + subchannel_set_id: u8, + device_number: u16, +} + +impl Device { + pub fn new(subchannel_set_id: u8, device_number: u16) -> anyhow::Result { + if subchannel_set_id > SUBCHANNEL_SET_MAX { + return Err(anyhow!( + "Subchannel set ID {:?} should be in range [0..{}]", + subchannel_set_id, + SUBCHANNEL_SET_MAX + )); + } + + Ok(Device { + subchannel_set_id, + device_number, + }) + } +} + +impl FromStr for Device { + type Err = anyhow::Error; + + fn from_str(s: &str) -> anyhow::Result { + let split: Vec<&str> = s.split('.').collect(); + if split.len() != 3 { + return Err(anyhow!( + "Wrong bus format. It needs to be in the form 0.., got {:?}", + s + )); + } + + if split[0] != "0" { + return Err(anyhow!( + "Wrong bus format. First digit needs to be 0, but is {:?}", + split[0] + )); + } + + let subchannel_set_id = match split[1].parse::() { + Ok(id) => id, + Err(_) => { + return Err(anyhow!( + "Wrong bus format. Second digit needs to be 0-3, but is {:?}", + split[1] + )) + } + }; + + let device_number = match u16::from_str_radix(split[2], 16) { + Ok(id) => id, + Err(_) => { + return Err(anyhow!( + "Wrong bus format. Third digit needs to be 0-ffff, but is {:?}", + split[2] + )) + } + }; + + Device::new(subchannel_set_id, device_number) + } +} + +impl fmt::Display for Device { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!(f, "0.{}.{:04x}", self.subchannel_set_id, self.device_number) + } +} + +#[cfg(test)] +mod tests { + use crate::ccw::Device; + use std::str::FromStr; + + #[test] + fn test_new_device() { + // Valid devices + let device = Device::new(0, 0).unwrap(); + assert_eq!(format!("{}", device), "0.0.0000"); + + let device = Device::new(3, 0xffff).unwrap(); + assert_eq!(format!("{}", device), "0.3.ffff"); + + // Invalid device + let device = Device::new(4, 0); + assert!(device.is_err()); + } + + #[test] + fn test_device_from_str() { + // Valid devices + let device = Device::from_str("0.0.0").unwrap(); + assert_eq!(format!("{}", device), "0.0.0000"); + + let device = Device::from_str("0.0.0000").unwrap(); + assert_eq!(format!("{}", device), "0.0.0000"); + + let device = Device::from_str("0.3.ffff").unwrap(); + assert_eq!(format!("{}", device), "0.3.ffff"); + + // Invalid devices + let device = Device::from_str("0.0"); + assert!(device.is_err()); + + let device = Device::from_str("1.0.0"); + assert!(device.is_err()); + + let device = Device::from_str("0.not_a_subchannel_set_id.0"); + assert!(device.is_err()); + + let device = Device::from_str("0.0.not_a_device_number"); + assert!(device.is_err()); + } +} diff --git a/src/agent/src/device.rs b/src/agent/src/device.rs index 5af1f649c..60a02147a 100644 --- a/src/agent/src/device.rs +++ b/src/agent/src/device.rs @@ -14,8 +14,13 @@ use std::str::FromStr; use std::sync::Arc; use tokio::sync::Mutex; +#[cfg(target_arch = "s390x")] +use crate::ccw; use crate::linux_abi::*; -use crate::mount::{DRIVER_BLK_TYPE, DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_SCSI_TYPE}; +use crate::mount::{ + DRIVER_BLK_CCW_TYPE, DRIVER_BLK_TYPE, DRIVER_MMIO_BLK_TYPE, DRIVER_NVDIMM_TYPE, + DRIVER_SCSI_TYPE, +}; use crate::pci; use crate::sandbox::Sandbox; use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher}; @@ -163,6 +168,47 @@ pub async fn get_virtio_blk_pci_device_name( Ok(format!("{}/{}", SYSTEM_DEV_PATH, &uev.devname)) } +#[cfg(target_arch = "s390x")] +#[derive(Debug)] +struct VirtioBlkCCWMatcher { + rex: Regex, +} + +#[cfg(target_arch = "s390x")] +impl VirtioBlkCCWMatcher { + fn new(root_bus_path: &str, device: &ccw::Device) -> Self { + let re = format!( + r"^{}/0\.[0-3]\.[0-9a-f]{{1,4}}/{}/virtio[0-9]+/block/", + root_bus_path, device + ); + VirtioBlkCCWMatcher { + rex: Regex::new(&re).unwrap(), + } + } +} + +#[cfg(target_arch = "s390x")] +impl UeventMatcher for VirtioBlkCCWMatcher { + fn is_match(&self, uev: &Uevent) -> bool { + uev.action == "add" && self.rex.is_match(&uev.devpath) && !uev.devname.is_empty() + } +} + +#[cfg(target_arch = "s390x")] +#[instrument] +pub async fn get_virtio_blk_ccw_device_name( + sandbox: &Arc>, + device: &ccw::Device, +) -> Result { + let matcher = VirtioBlkCCWMatcher::new(&create_ccw_root_bus_path(), device); + let uev = wait_for_uevent(sandbox, matcher).await?; + let devname = uev.devname; + return match Path::new(SYSTEM_DEV_PATH).join(&devname).to_str() { + Some(path) => Ok(String::from(path)), + None => Err(anyhow!("CCW device name {} is not valid UTF-8", &devname)), + }; +} + #[derive(Debug)] struct PmemBlockMatcher { suffix: String, @@ -352,6 +398,32 @@ async fn virtio_blk_device_handler( update_spec_device_list(&dev, spec, devidx) } +// device.id should be a CCW path string +#[cfg(target_arch = "s390x")] +#[instrument] +async fn virtio_blk_ccw_device_handler( + device: &Device, + spec: &mut Spec, + sandbox: &Arc>, + devidx: &DevIndex, +) -> Result<()> { + let mut dev = device.clone(); + let ccw_device = ccw::Device::from_str(&device.id)?; + dev.vm_path = get_virtio_blk_ccw_device_name(sandbox, &ccw_device).await?; + update_spec_device_list(&dev, spec, devidx) +} + +#[cfg(not(target_arch = "s390x"))] +#[instrument] +async fn virtio_blk_ccw_device_handler( + _: &Device, + _: &mut Spec, + _: &Arc>, + _: &DevIndex, +) -> Result<()> { + Err(anyhow!("CCW is only supported on s390x")) +} + // device.Id should be the SCSI address of the disk in the format "scsiID:lunID" #[instrument] async fn virtio_scsi_device_handler( @@ -444,6 +516,7 @@ async fn add_device( match device.field_type.as_str() { DRIVER_BLK_TYPE => virtio_blk_device_handler(device, spec, sandbox, devidx).await, + DRIVER_BLK_CCW_TYPE => virtio_blk_ccw_device_handler(device, spec, sandbox, devidx).await, DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, spec, sandbox, devidx).await, DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, spec, sandbox, devidx).await, DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, spec, sandbox, devidx).await, @@ -906,6 +979,66 @@ mod tests { assert!(!matcher_a.is_match(&uev_b)); } + #[cfg(target_arch = "s390x")] + #[tokio::test] + async fn test_virtio_blk_ccw_matcher() { + let root_bus = create_ccw_root_bus_path(); + let subsystem = "block"; + let devname = "vda"; + let relpath = "0.0.0002"; + + let mut uev = crate::uevent::Uevent::default(); + uev.action = crate::linux_abi::U_EVENT_ACTION_ADD.to_string(); + uev.subsystem = subsystem.to_string(); + uev.devname = devname.to_string(); + uev.devpath = format!( + "{}/0.0.0001/{}/virtio1/{}/{}", + root_bus, relpath, subsystem, devname + ); + + // Valid path + let device = ccw::Device::from_str(relpath).unwrap(); + let matcher = VirtioBlkCCWMatcher::new(&root_bus, &device); + assert!(matcher.is_match(&uev)); + + // Invalid paths + uev.devpath = format!( + "{}/0.0.0001/0.0.0003/virtio1/{}/{}", + root_bus, subsystem, devname + ); + assert!(!matcher.is_match(&uev)); + + uev.devpath = format!("0.0.0001/{}/virtio1/{}/{}", relpath, subsystem, devname); + assert!(!matcher.is_match(&uev)); + + uev.devpath = format!( + "{}/0.0.0001/{}/virtio/{}/{}", + root_bus, relpath, subsystem, devname + ); + assert!(!matcher.is_match(&uev)); + + uev.devpath = format!("{}/0.0.0001/{}/virtio1", root_bus, relpath); + assert!(!matcher.is_match(&uev)); + + uev.devpath = format!( + "{}/1.0.0001/{}/virtio1/{}/{}", + root_bus, relpath, subsystem, devname + ); + assert!(!matcher.is_match(&uev)); + + uev.devpath = format!( + "{}/0.4.0001/{}/virtio1/{}/{}", + root_bus, relpath, subsystem, devname + ); + assert!(!matcher.is_match(&uev)); + + uev.devpath = format!( + "{}/0.0.10000/{}/virtio1/{}/{}", + root_bus, relpath, subsystem, devname + ); + assert!(!matcher.is_match(&uev)); + } + #[tokio::test] async fn test_scsi_block_matcher() { let root_bus = create_pci_root_bus_path(); diff --git a/src/agent/src/linux_abi.rs b/src/agent/src/linux_abi.rs index de695a129..08600d9f7 100644 --- a/src/agent/src/linux_abi.rs +++ b/src/agent/src/linux_abi.rs @@ -65,6 +65,10 @@ pub fn create_pci_root_bus_path() -> String { ret } +#[cfg(target_arch = "s390x")] +pub fn create_ccw_root_bus_path() -> String { + String::from("/devices/css0") +} // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt // The Linux kernel's core ACPI subsystem creates struct acpi_device // objects for ACPI namespace objects representing devices, power resources diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 7ec13405c..848096142 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -34,6 +34,8 @@ use std::process::exit; use std::sync::Arc; use tracing::{instrument, span}; +#[cfg(target_arch = "s390x")] +mod ccw; mod config; mod console; mod device; diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index 26d3cceb7..ef96728cf 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -31,6 +31,8 @@ use crate::linux_abi::*; use crate::pci; use crate::protocols::agent::Storage; use crate::Sandbox; +#[cfg(target_arch = "s390x")] +use crate::{ccw, device::get_virtio_blk_ccw_device_name}; use anyhow::{anyhow, Context, Result}; use slog::Logger; use tracing::instrument; @@ -38,6 +40,7 @@ use tracing::instrument; pub const DRIVER_9P_TYPE: &str = "9p"; pub const DRIVER_VIRTIOFS_TYPE: &str = "virtio-fs"; pub const DRIVER_BLK_TYPE: &str = "blk"; +pub const DRIVER_BLK_CCW_TYPE: &str = "blk-ccw"; pub const DRIVER_MMIO_BLK_TYPE: &str = "mmioblk"; pub const DRIVER_SCSI_TYPE: &str = "scsi"; pub const DRIVER_NVDIMM_TYPE: &str = "nvdimm"; @@ -389,6 +392,31 @@ async fn virtio_blk_storage_handler( common_storage_handler(logger, &storage) } +// virtio_blk_ccw_storage_handler handles storage for the blk-ccw driver (s390x) +#[cfg(target_arch = "s390x")] +#[instrument] +async fn virtio_blk_ccw_storage_handler( + logger: &Logger, + storage: &Storage, + sandbox: Arc>, +) -> Result { + let mut storage = storage.clone(); + let ccw_device = ccw::Device::from_str(&storage.source)?; + let dev_path = get_virtio_blk_ccw_device_name(&sandbox, &ccw_device).await?; + storage.source = dev_path; + common_storage_handler(logger, &storage) +} + +#[cfg(not(target_arch = "s390x"))] +#[instrument] +async fn virtio_blk_ccw_storage_handler( + _: &Logger, + _: &Storage, + _: Arc>, +) -> Result { + Err(anyhow!("CCW is only supported on s390x")) +} + // virtio_scsi_storage_handler handles the storage for scsi driver. #[instrument] async fn virtio_scsi_storage_handler( @@ -557,6 +585,9 @@ pub async fn add_storages( let res = match handler_name.as_str() { DRIVER_BLK_TYPE => virtio_blk_storage_handler(&logger, &storage, sandbox.clone()).await, + DRIVER_BLK_CCW_TYPE => { + virtio_blk_ccw_storage_handler(&logger, &storage, sandbox.clone()).await + } DRIVER_9P_TYPE => virtio9p_storage_handler(&logger, &storage, sandbox.clone()).await, DRIVER_VIRTIOFS_TYPE => { virtiofs_storage_handler(&logger, &storage, sandbox.clone()).await diff --git a/src/agent/src/uevent.rs b/src/agent/src/uevent.rs index e7d7b3bf6..4b391c869 100644 --- a/src/agent/src/uevent.rs +++ b/src/agent/src/uevent.rs @@ -114,7 +114,7 @@ pub async fn wait_for_uevent( let mut sb = sandbox.lock().await; for uev in sb.uevent_map.values() { if matcher.is_match(uev) { - info!(sl!(), "Device {:?} found in pci device map", uev); + info!(sl!(), "Device {:?} found in device map", uev); return Ok(uev.clone()); } }