agent: Added test-case for handle_cdi_devices

We are generating a simple CDI spec with device and
global containerEdits to test the CDI crate.

Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>

src/agent/Cargo.lock

@@ -886,7 +886,7 @@ dependencies = [
 [[package]]
 name = "cdi"
 version = "0.1.0"
-source = "git+https://github.com/cncf-tags/container-device-interface-rs?rev=19763bcf0de62599ce8b88558418b4dac1154745#19763bcf0de62599ce8b88558418b4dac1154745"
+source = "git+https://github.com/cncf-tags/container-device-interface-rs?rev=fba5677a8e7cc962fc6e495fcec98d7d765e332a#fba5677a8e7cc962fc6e495fcec98d7d765e332a"
 dependencies = [
  "anyhow",
  "clap 4.5.13",
@@ -5411,7 +5411,7 @@ dependencies = [
  "aes",
  "aes-gcm",
  "anyhow",
- "base64 0.22.1",
+ "base64 0.21.7",
  "block-padding",
  "blowfish",
  "buffered-reader",

src/agent/Cargo.toml

@@ -84,8 +84,7 @@ regorus = { version = "0.1.4", default-features = false, features = [
     "arc",
     "regex",
 ], optional = true }
-#cdi = { git = "https://github.com/cncf-tags/container-device-interface-rs", rev = "19763bcf0de62599ce8b88558418b4dac1154745" }
+cdi = { git = "https://github.com/cncf-tags/container-device-interface-rs", rev = "fba5677a8e7cc962fc6e495fcec98d7d765e332a" }
-cdi = { git = "https://github.com/zvonkok/container-device-interface-rs", branch = "rename-with-auto-refresh" }
 
 [dev-dependencies]
 tempfile = "3.1.0"

src/agent/src/device/mod.rs

@@ -12,7 +12,7 @@ use crate::pci;
 use crate::sandbox::Sandbox;
 use anyhow::{anyhow, Context, Result};
 use cdi::annotations::parse_annotations;
-use cdi::cache::{new_cache, CdiOption, with_auto_refresh};
+use cdi::cache::{new_cache, with_auto_refresh, CdiOption};
 use cdi::spec_dirs::with_spec_dirs;
 use kata_types::device::DeviceHandlerManager;
 use nix::sys::stat;
@@ -40,8 +40,6 @@ pub mod vfio_device_handler;
 
 pub const BLOCK: &str = "block";
 
-
-
 #[derive(Debug, Clone)]
 pub struct DeviceInfo {
     // Device type, "b" for block device and "c" for character device
@@ -246,7 +244,12 @@ pub async fn add_devices(
 }
 
 #[instrument]
-pub async fn handle_cdi_devices(logger: &Logger, spec: &mut Spec, spec_dir: &str, cdi_timeout: u64) -> Result<()> {
+pub async fn handle_cdi_devices(
+    logger: &Logger,
+    spec: &mut Spec,
+    spec_dir: &str,
+    cdi_timeout: u64,
+) -> Result<()> {
     if let Some(container_type) = spec
         .annotations()
         .as_ref()
@@ -265,11 +268,7 @@ pub async fn handle_cdi_devices(logger: &Logger, spec: &mut Spec, spec_dir: &str
     }
     // Explicitly set the cache options to disable auto-refresh and
     // to use the single spec dir "/var/run/cdi" for tests it can be overridden
-    let options: Vec<CdiOption> = vec![
+    let options: Vec<CdiOption> = vec![with_auto_refresh(false), with_spec_dirs(&[spec_dir])];
-        with_auto_refresh(true),
-        with_spec_dirs(&[spec_dir]),
-    ];
-
     let cache: Arc<std::sync::Mutex<cdi::cache::Cache>> = new_cache(options);
 
     for _ in 0..=cdi_timeout {
@@ -277,6 +276,12 @@ pub async fn handle_cdi_devices(logger: &Logger, spec: &mut Spec, spec_dir: &str
             // Lock cache within this scope, std::sync::Mutex has no Send
             // and await will not work with time::sleep
             let mut cache = cache.lock().unwrap();
+            match cache.refresh() {
+                Ok(_) => {}
+                Err(e) => {
+                    return Err(anyhow!("error refreshing cache: {:?}", e));
+                }
+            }
             cache.inject_devices(Some(spec), devices.clone())
         };
 
@@ -290,6 +295,7 @@ pub async fn handle_cdi_devices(logger: &Logger, spec: &mut Spec, spec_dir: &str
             }
             Err(e) => {
                 info!(logger, "error injecting devices: {:?}", e);
+                println!("error injecting devices: {:?}", e);
             }
         }
         time::sleep(Duration::from_millis(1000)).await;
@@ -1175,7 +1181,6 @@ mod tests {
 
     #[tokio::test]
     async fn test_handle_cdi_devices() {
-
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut spec = Spec::default();
 
@@ -1187,47 +1192,81 @@ mod tests {
         );
         spec.set_annotations(Some(annotations));
 
-
         // create a file in /tmp/cdi with nvidia.json content
         let cdi_dir = PathBuf::from("/tmp/cdi");
         let cdi_file = cdi_dir.join("kata.json");
-        let cdi_content = r#"{
+
-            "cdiVersion": "0.7.0",
+        let cdi_version = "0.6.0";
-            "kind": "kata.com/gpu",
+        let kind = "kata.com/gpu";
+        let device_name = "0";
+        let annotation_whatever = "false";
+        let annotation_whenever = "true";
+        let inner_env = "TEST_INNER_ENV=TEST_INNER_ENV_VALUE";
+        let outer_env = "TEST_OUTER_ENV=TEST_OUTER_ENV_VALUE";
+        let inner_device = "/dev/zero";
+        let outer_device = "/dev/null";
+
+        let cdi_content = format!(
+            r#"{{
+            "cdiVersion": "{cdi_version}",
+            "kind": "{kind}",
             "devices": [
-                {
+                {{
-                    "name": "0",
+                    "name": "{device_name}",
-                    "annotations": {
+                    "annotations": {{
-                        "whatever": "false",
+                        "whatever": "{annotation_whatever}",
-                        "whenever": "true"
+                        "whenever": "{annotation_whenever}"
-                    },
+                    }},
-                    "containerEdits": {
+                    "containerEdits": {{
+                        "env": [
+                            "{inner_env}"
+                        ],
                         "deviceNodes": [
-                            {
+                            {{
-                                "path": "/dev/rtc0"
+                                "path": "{inner_device}"
-                            }
+                            }}
                         ]
-                    }
+                    }}
-                }
+                }}
+            ],
+            "containerEdits": {{
+                "env": [
+                    "{outer_env}"
+                ],
+                "deviceNodes": [
+                    {{
+                        "path": "{outer_device}"
+                    }}
                 ]
-        }
+            }}
-        "#;
+        }}"#
+        );
 
         fs::create_dir_all(&cdi_dir).unwrap();
         fs::write(&cdi_file, cdi_content).unwrap();
 
-
+        let res = handle_cdi_devices(&logger, &mut spec, "/tmp/cdi", 0).await;
-        let res = handle_cdi_devices(&logger, &mut spec, "/tmp/cdi", 1).await;
+        println!("modfied spec {:?}", spec);
         assert!(res.is_ok(), "{}", res.err().unwrap());
 
+        let linux = spec.linux().as_ref().unwrap();
+        let devices = linux
+            .resources()
+            .as_ref()
+            .unwrap()
+            .devices()
+            .as_ref()
+            .unwrap();
+        assert_eq!(devices.len(), 2);
 
-        //let linux = spec.linux().as_ref().unwrap();
+        let env = spec.process().as_ref().unwrap().env().as_ref().unwrap();
-        //let devices = linux.resources().as_ref().unwrap().devices().as_ref().unwrap();
-        //assert_eq!(devices.len(), 1);
 
-        //let dev = &devices[0];
+        // find string TEST_OUTER_ENV in evn
-        //assert_eq!(dev.major(), 10);
+        let outer_env = env.iter().find(|e| e.starts_with("TEST_OUTER_ENV"));
-        //assert_eq!(dev.minor(), 200);
+        assert!(outer_env.is_some(), "TEST_OUTER_ENV not found in env");
 
+        // find TEST_INNER_ENV in env
+        let inner_env = env.iter().find(|e| e.starts_with("TEST_INNER_ENV"));
+        assert!(inner_env.is_some(), "TEST_INNER_ENV not found in env");
     }
 }