CCv0: Merge main into CCv0 branch

Merge remote-tracking branch 'upstream/main' into CCv0

Fixes: #4090
Signed-off-by: Megan Wright <megan.wright@ibm.com>

docs/Limitations.md

@@ -46,7 +46,7 @@ The following link shows the latest list of limitations:
 # Contributing
 
 If you would like to work on resolving a limitation, please refer to the
-[contributors guide](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md).
+[contributors guide](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md).
 If you wish to raise an issue for a new limitation, either
 [raise an issue directly on the runtime](https://github.com/kata-containers/kata-containers/issues/new)
 or see the

src/agent/Makefile

@@ -14,10 +14,6 @@ PROJECT_COMPONENT = kata-agent
 
 TARGET = $(PROJECT_COMPONENT)
 
-SOURCES := \
-  $(shell find . 2>&1 | grep -E '.*\.rs$$') \
-  Cargo.toml
-
 VERSION_FILE := ./VERSION
 VERSION := $(shell grep -v ^\# $(VERSION_FILE))
 COMMIT_NO := $(shell git rev-parse HEAD 2>/dev/null || true)
@@ -38,7 +34,7 @@ ifeq ($(SECCOMP),yes)
 endif
 
 ifneq ($(EXTRA_RUSTFEATURES),)
-    override EXTRA_RUSTFEATURES := --features $(EXTRA_RUSTFEATURES)
+    override EXTRA_RUSTFEATURES := --features "$(EXTRA_RUSTFEATURES)"
 endif
 
 include ../../utils.mk
@@ -108,14 +104,14 @@ $(TARGET): $(GENERATED_CODE) logging-crate-tests $(TARGET_PATH)
 logging-crate-tests:
 	make -C $(CWD)/../libs/logging
 
-$(TARGET_PATH): $(SOURCES) | show-summary
+$(TARGET_PATH): show-summary
 	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
 $(GENERATED_FILES): %: %.in
 	@sed $(foreach r,$(GENERATED_REPLACEMENTS),-e 's|@$r@|$($r)|g') "$<" > "$@"
 
 ##TARGET optimize: optimized  build
-optimize: $(SOURCES) | show-summary show-header
+optimize: show-summary show-header
 	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
 ##TARGET install: install agent

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -593,9 +593,9 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
 
         let h = lines_to_map(&cpuacct.stat);
         let usage_in_usermode =
-            (((*h.get("user").unwrap() * NANO_PER_SECOND) as f64) / *CLOCK_TICKS) as u64;
+            (((*h.get("user").unwrap_or(&0) * NANO_PER_SECOND) as f64) / *CLOCK_TICKS) as u64;
         let usage_in_kernelmode =
-            (((*h.get("system").unwrap() * NANO_PER_SECOND) as f64) / *CLOCK_TICKS) as u64;
+            (((*h.get("system").unwrap_or(&0) * NANO_PER_SECOND) as f64) / *CLOCK_TICKS) as u64;
 
         let total_usage = cpuacct.usage;
 
@@ -626,9 +626,9 @@ fn get_cpuacct_stats(cg: &cgroups::Cgroup) -> SingularPtrField<CpuUsage> {
     let cpu_controller: &CpuController = get_controller_or_return_singular_none!(cg);
     let stat = cpu_controller.cpu().stat;
     let h = lines_to_map(&stat);
-    let usage_in_usermode = *h.get("user_usec").unwrap();
+    let usage_in_usermode = *h.get("user_usec").unwrap_or(&0);
-    let usage_in_kernelmode = *h.get("system_usec").unwrap();
+    let usage_in_kernelmode = *h.get("system_usec").unwrap_or(&0);
-    let total_usage = *h.get("usage_usec").unwrap();
+    let total_usage = *h.get("usage_usec").unwrap_or(&0);
     let percpu_usage = vec![];
 
     SingularPtrField::some(CpuUsage {

src/agent/rustjail/src/lib.rs

@@ -518,6 +518,7 @@ pub fn grpc_to_oci(grpc: &grpc::Spec) -> oci::Spec {
 
 #[cfg(test)]
 mod tests {
+    use super::*;
     #[macro_export]
     macro_rules! skip_if_not_root {
         () => {
@@ -527,4 +528,162 @@ mod tests {
             }
         };
     }
+
+    #[test]
+    fn test_process_grpc_to_oci() {
+        #[derive(Debug)]
+        struct TestData {
+            grpcproc: grpc::Process,
+            result: oci::Process,
+        }
+
+        let tests = &[
+            TestData {
+                // All fields specified
+                grpcproc: grpc::Process {
+                    Terminal: true,
+                    ConsoleSize: protobuf::SingularPtrField::<grpc::Box>::some(grpc::Box {
+                        Height: 123,
+                        Width: 456,
+                        ..Default::default()
+                    }),
+                    User: protobuf::SingularPtrField::<grpc::User>::some(grpc::User {
+                        UID: 1234,
+                        GID: 5678,
+                        AdditionalGids: Vec::from([910, 1112]),
+                        Username: String::from("username"),
+                        ..Default::default()
+                    }),
+                    Args: protobuf::RepeatedField::from(Vec::from([
+                        String::from("arg1"),
+                        String::from("arg2"),
+                    ])),
+                    Env: protobuf::RepeatedField::from(Vec::from([String::from("env")])),
+                    Cwd: String::from("cwd"),
+                    Capabilities: protobuf::SingularPtrField::some(grpc::LinuxCapabilities {
+                        Bounding: protobuf::RepeatedField::from(Vec::from([String::from("bnd")])),
+                        Effective: protobuf::RepeatedField::from(Vec::from([String::from("eff")])),
+                        Inheritable: protobuf::RepeatedField::from(Vec::from([String::from(
+                            "inher",
+                        )])),
+                        Permitted: protobuf::RepeatedField::from(Vec::from([String::from("perm")])),
+                        Ambient: protobuf::RepeatedField::from(Vec::from([String::from("amb")])),
+                        ..Default::default()
+                    }),
+                    Rlimits: protobuf::RepeatedField::from(Vec::from([
+                        grpc::POSIXRlimit {
+                            Type: String::from("r#type"),
+                            Hard: 123,
+                            Soft: 456,
+                            ..Default::default()
+                        },
+                        grpc::POSIXRlimit {
+                            Type: String::from("r#type2"),
+                            Hard: 789,
+                            Soft: 1011,
+                            ..Default::default()
+                        },
+                    ])),
+                    NoNewPrivileges: true,
+                    ApparmorProfile: String::from("apparmor profile"),
+                    OOMScoreAdj: 123456,
+                    SelinuxLabel: String::from("Selinux Label"),
+                    ..Default::default()
+                },
+                result: oci::Process {
+                    terminal: true,
+                    console_size: Some(oci::Box {
+                        height: 123,
+                        width: 456,
+                    }),
+                    user: oci::User {
+                        uid: 1234,
+                        gid: 5678,
+                        additional_gids: Vec::from([910, 1112]),
+                        username: String::from("username"),
+                    },
+                    args: Vec::from([String::from("arg1"), String::from("arg2")]),
+                    env: Vec::from([String::from("env")]),
+                    cwd: String::from("cwd"),
+                    capabilities: Some(oci::LinuxCapabilities {
+                        bounding: Vec::from([String::from("bnd")]),
+                        effective: Vec::from([String::from("eff")]),
+                        inheritable: Vec::from([String::from("inher")]),
+                        permitted: Vec::from([String::from("perm")]),
+                        ambient: Vec::from([String::from("amb")]),
+                    }),
+                    rlimits: Vec::from([
+                        oci::PosixRlimit {
+                            r#type: String::from("r#type"),
+                            hard: 123,
+                            soft: 456,
+                        },
+                        oci::PosixRlimit {
+                            r#type: String::from("r#type2"),
+                            hard: 789,
+                            soft: 1011,
+                        },
+                    ]),
+                    no_new_privileges: true,
+                    apparmor_profile: String::from("apparmor profile"),
+                    oom_score_adj: Some(123456),
+                    selinux_label: String::from("Selinux Label"),
+                },
+            },
+            TestData {
+                // None ConsoleSize
+                grpcproc: grpc::Process {
+                    ConsoleSize: protobuf::SingularPtrField::<grpc::Box>::none(),
+                    OOMScoreAdj: 0,
+                    ..Default::default()
+                },
+                result: oci::Process {
+                    console_size: None,
+                    oom_score_adj: Some(0),
+                    ..Default::default()
+                },
+            },
+            TestData {
+                // None User
+                grpcproc: grpc::Process {
+                    User: protobuf::SingularPtrField::<grpc::User>::none(),
+                    OOMScoreAdj: 0,
+                    ..Default::default()
+                },
+                result: oci::Process {
+                    user: oci::User {
+                        uid: 0,
+                        gid: 0,
+                        additional_gids: vec![],
+                        username: String::from(""),
+                    },
+                    oom_score_adj: Some(0),
+                    ..Default::default()
+                },
+            },
+            TestData {
+                // None Capabilities
+                grpcproc: grpc::Process {
+                    Capabilities: protobuf::SingularPtrField::none(),
+                    OOMScoreAdj: 0,
+                    ..Default::default()
+                },
+                result: oci::Process {
+                    capabilities: None,
+                    oom_score_adj: Some(0),
+                    ..Default::default()
+                },
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let result = process_grpc_to_oci(&d.grpcproc);
+
+            let msg = format!("{}, result: {:?}", msg, result);
+
+            assert_eq!(d.result, result, "{}", msg);
+        }
+    }
 }

src/agent/src/mount.rs

@@ -1497,6 +1497,130 @@ mod tests {
         assert!(testfile.is_file());
     }
 
+    #[test]
+    fn test_mount_storage() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            test_user: TestUserType,
+            storage: Storage,
+            error_contains: &'a str,
+
+            make_source_dir: bool,
+            make_mount_dir: bool,
+            deny_mount_permission: bool,
+        }
+
+        impl Default for TestData<'_> {
+            fn default() -> Self {
+                TestData {
+                    test_user: TestUserType::Any,
+                    storage: Storage {
+                        mount_point: "mnt".to_string(),
+                        source: "src".to_string(),
+                        fstype: "tmpfs".to_string(),
+                        ..Default::default()
+                    },
+                    make_source_dir: true,
+                    make_mount_dir: false,
+                    deny_mount_permission: false,
+                    error_contains: "",
+                }
+            }
+        }
+
+        let tests = &[
+            TestData {
+                test_user: TestUserType::NonRootOnly,
+                error_contains: "EPERM: Operation not permitted",
+                ..Default::default()
+            },
+            TestData {
+                test_user: TestUserType::RootOnly,
+                ..Default::default()
+            },
+            TestData {
+                storage: Storage {
+                    mount_point: "mnt".to_string(),
+                    source: "src".to_string(),
+                    fstype: "bind".to_string(),
+                    ..Default::default()
+                },
+                make_source_dir: false,
+                make_mount_dir: true,
+                error_contains: "Could not create mountpoint",
+                ..Default::default()
+            },
+            TestData {
+                test_user: TestUserType::NonRootOnly,
+                deny_mount_permission: true,
+                error_contains: "Could not create mountpoint",
+                ..Default::default()
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+            if d.test_user == TestUserType::RootOnly {
+                skip_loop_if_not_root!(msg);
+            } else if d.test_user == TestUserType::NonRootOnly {
+                skip_loop_if_root!(msg);
+            }
+
+            let drain = slog::Discard;
+            let logger = slog::Logger::root(drain, o!());
+
+            let tempdir = tempdir().unwrap();
+
+            let source = tempdir.path().join(&d.storage.source);
+            let mount_point = tempdir.path().join(&d.storage.mount_point);
+
+            let storage = Storage {
+                source: source.to_str().unwrap().to_string(),
+                mount_point: mount_point.to_str().unwrap().to_string(),
+                ..d.storage.clone()
+            };
+
+            if d.make_source_dir {
+                fs::create_dir_all(&storage.source).unwrap();
+            }
+            if d.make_mount_dir {
+                fs::create_dir_all(&storage.mount_point).unwrap();
+            }
+
+            if d.deny_mount_permission {
+                fs::set_permissions(
+                    mount_point.parent().unwrap(),
+                    fs::Permissions::from_mode(0o000),
+                )
+                .unwrap();
+            }
+
+            let result = mount_storage(&logger, &storage);
+
+            // restore permissions so tempdir can be cleaned up
+            if d.deny_mount_permission {
+                fs::set_permissions(
+                    mount_point.parent().unwrap(),
+                    fs::Permissions::from_mode(0o755),
+                )
+                .unwrap();
+            }
+
+            if result.is_ok() {
+                nix::mount::umount(&mount_point).unwrap();
+            }
+
+            let msg = format!("{}: result: {:?}", msg, result);
+            if d.error_contains.is_empty() {
+                assert!(result.is_ok(), "{}", msg);
+            } else {
+                assert!(result.is_err(), "{}", msg);
+                let error_msg = format!("{}", result.unwrap_err());
+                assert!(error_msg.contains(d.error_contains), "{}", msg);
+            }
+        }
+    }
+
     #[test]
     fn test_get_pagesize_and_size_from_option() {
         let expected_pagesize = 2048;
@@ -1552,4 +1676,55 @@ mod tests {
             }
         }
     }
+
+    #[test]
+    fn test_parse_mount_flags_and_options() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            options_vec: Vec<&'a str>,
+            result: (MsFlags, &'a str),
+        }
+
+        let tests = &[
+            TestData {
+                options_vec: vec![],
+                result: (MsFlags::empty(), ""),
+            },
+            TestData {
+                options_vec: vec!["ro"],
+                result: (MsFlags::MS_RDONLY, ""),
+            },
+            TestData {
+                options_vec: vec!["rw"],
+                result: (MsFlags::empty(), ""),
+            },
+            TestData {
+                options_vec: vec!["ro", "rw"],
+                result: (MsFlags::empty(), ""),
+            },
+            TestData {
+                options_vec: vec!["ro", "nodev"],
+                result: (MsFlags::MS_RDONLY | MsFlags::MS_NODEV, ""),
+            },
+            TestData {
+                options_vec: vec!["option1", "nodev", "option2"],
+                result: (MsFlags::MS_NODEV, "option1,option2"),
+            },
+            TestData {
+                options_vec: vec!["rbind", "", "ro"],
+                result: (MsFlags::MS_BIND | MsFlags::MS_REC | MsFlags::MS_RDONLY, ""),
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let result = parse_mount_flags_and_options(d.options_vec.clone());
+
+            let msg = format!("{}: result: {:?}", msg, result);
+
+            let expected_result = (d.result.0, d.result.1.to_owned());
+            assert_eq!(expected_result, result, "{}", msg);
+        }
+    }
 }

src/agent/src/rpc.rs

@@ -88,6 +88,7 @@ const MODPROBE_PATH: &str = "/sbin/modprobe";
 const ANNO_K8S_IMAGE_NAME: &str = "io.kubernetes.cri.image-name";
 const CONFIG_JSON: &str = "config.json";
 
+const ERR_CANNOT_GET_WRITER: &str = "Cannot get writer";
 const ERR_INVALID_BLOCK_SIZE: &str = "Invalid block size";
 
 // Convenience macro to obtain the scope logger
@@ -596,7 +597,7 @@ impl AgentService {
             }
         };
 
-        let writer = writer.ok_or_else(|| anyhow!("cannot get writer"))?;
+        let writer = writer.ok_or_else(|| anyhow!(ERR_CANNOT_GET_WRITER))?;
         writer.lock().await.write_all(req.data.as_slice()).await?;
 
         let mut resp = WriteStreamResponse::new();
@@ -1961,7 +1962,7 @@ mod tests {
     use super::*;
     use crate::protocols::agent_ttrpc::AgentService as _;
     use oci::{Hook, Hooks};
-    use tempfile::tempdir;
+    use tempfile::{tempdir, TempDir};
     use ttrpc::{r#async::TtrpcContext, MessageHeader};
 
     // Parameters:
@@ -1999,6 +2000,44 @@ mod tests {
         }
     }
 
+    fn create_dummy_opts() -> CreateOpts {
+        let root = Root {
+            path: String::from("/"),
+            ..Default::default()
+        };
+
+        let spec = Spec {
+            linux: Some(oci::Linux::default()),
+            root: Some(root),
+            ..Default::default()
+        };
+
+        CreateOpts {
+            cgroup_name: "".to_string(),
+            use_systemd_cgroup: false,
+            no_pivot_root: false,
+            no_new_keyring: false,
+            spec: Some(spec),
+            rootless_euid: false,
+            rootless_cgroup: false,
+        }
+    }
+
+    fn create_linuxcontainer() -> (LinuxContainer, TempDir) {
+        let dir = tempdir().expect("failed to make tempdir");
+
+        (
+            LinuxContainer::new(
+                "some_id",
+                dir.path().join("rootfs").to_str().unwrap(),
+                create_dummy_opts(),
+                &slog_scope::logger(),
+            )
+            .unwrap(),
+            dir,
+        )
+    }
+
     #[test]
     fn test_load_kernel_module() {
         let mut m = protocols::agent::KernelModule {
@@ -2091,6 +2130,149 @@ mod tests {
         assert!(result.is_err(), "expected add arp neighbors to fail");
     }
 
+    #[tokio::test]
+    async fn test_do_write_stream() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            create_container: bool,
+            has_fd: bool,
+            has_tty: bool,
+            break_pipe: bool,
+
+            container_id: &'a str,
+            exec_id: &'a str,
+            data: Vec<u8>,
+            result: Result<protocols::agent::WriteStreamResponse>,
+        }
+
+        impl Default for TestData<'_> {
+            fn default() -> Self {
+                TestData {
+                    create_container: true,
+                    has_fd: true,
+                    has_tty: true,
+                    break_pipe: false,
+
+                    container_id: "1",
+                    exec_id: "2",
+                    data: vec![1, 2, 3],
+                    result: Ok(WriteStreamResponse {
+                        len: 3,
+                        ..WriteStreamResponse::default()
+                    }),
+                }
+            }
+        }
+
+        let tests = &[
+            TestData {
+                ..Default::default()
+            },
+            TestData {
+                has_tty: false,
+                ..Default::default()
+            },
+            TestData {
+                break_pipe: true,
+                result: Err(anyhow!(std::io::Error::from_raw_os_error(libc::EPIPE))),
+                ..Default::default()
+            },
+            TestData {
+                create_container: false,
+                result: Err(anyhow!(crate::sandbox::ERR_INVALID_CONTAINER_ID)),
+                ..Default::default()
+            },
+            TestData {
+                container_id: "8181",
+                result: Err(anyhow!(crate::sandbox::ERR_INVALID_CONTAINER_ID)),
+                ..Default::default()
+            },
+            TestData {
+                data: vec![],
+                result: Ok(WriteStreamResponse {
+                    len: 0,
+                    ..WriteStreamResponse::default()
+                }),
+                ..Default::default()
+            },
+            TestData {
+                has_fd: false,
+                result: Err(anyhow!(ERR_CANNOT_GET_WRITER)),
+                ..Default::default()
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d);
+
+            let logger = slog::Logger::root(slog::Discard, o!());
+            let mut sandbox = Sandbox::new(&logger).unwrap();
+
+            let (rfd, wfd) = unistd::pipe().unwrap();
+            if d.break_pipe {
+                unistd::close(rfd).unwrap();
+            }
+
+            if d.create_container {
+                let (mut linux_container, _root) = create_linuxcontainer();
+                let exec_process_id = 2;
+
+                linux_container.id = "1".to_string();
+
+                let mut exec_process = Process::new(
+                    &logger,
+                    &oci::Process::default(),
+                    &exec_process_id.to_string(),
+                    false,
+                    1,
+                )
+                .unwrap();
+
+                let fd = {
+                    if d.has_fd {
+                        Some(wfd)
+                    } else {
+                        None
+                    }
+                };
+
+                if d.has_tty {
+                    exec_process.parent_stdin = None;
+                    exec_process.term_master = fd;
+                } else {
+                    exec_process.parent_stdin = fd;
+                    exec_process.term_master = None;
+                }
+                linux_container
+                    .processes
+                    .insert(exec_process_id, exec_process);
+
+                sandbox.add_container(linux_container);
+            }
+
+            let agent_service = Box::new(AgentService {
+                sandbox: Arc::new(Mutex::new(sandbox)),
+            });
+
+            let result = agent_service
+                .do_write_stream(protocols::agent::WriteStreamRequest {
+                    container_id: d.container_id.to_string(),
+                    exec_id: d.exec_id.to_string(),
+                    data: d.data.clone(),
+                    ..Default::default()
+                })
+                .await;
+
+            if !d.break_pipe {
+                unistd::close(rfd).unwrap();
+            }
+            unistd::close(wfd).unwrap();
+
+            let msg = format!("{}, result: {:?}", msg, result);
+            assert_result!(d.result, result, msg);
+        }
+    }
+
     #[tokio::test]
     async fn test_get_memory_info() {
         #[derive(Debug)]

src/agent/src/sandbox.rs

@@ -32,6 +32,8 @@ use tokio::sync::oneshot;
 use tokio::sync::Mutex;
 use tracing::instrument;
 
+pub const ERR_INVALID_CONTAINER_ID: &str = "Invalid container id";
+
 type UeventWatcher = (Box<dyn UeventMatcher>, oneshot::Sender<Uevent>);
 
 #[derive(Debug)]
@@ -234,7 +236,7 @@ impl Sandbox {
     pub fn find_container_process(&mut self, cid: &str, eid: &str) -> Result<&mut Process> {
         let ctr = self
             .get_container(cid)
-            .ok_or_else(|| anyhow!("Invalid container id"))?;
+            .ok_or_else(|| anyhow!(ERR_INVALID_CONTAINER_ID))?;
 
         if eid.is_empty() {
             return ctr

src/libs/protocols/.gitignore

@@ -1,6 +1,7 @@
 Cargo.lock
 src/agent.rs
 src/agent_ttrpc.rs
+src/csi.rs
 src/empty.rs
 src/health.rs
 src/health_ttrpc.rs

src/runtime/cmd/kata-monitor/main.go

@@ -175,6 +175,15 @@ func main() {
 }
 
 func indexPage(w http.ResponseWriter, r *http.Request) {
+	htmlResponse := kataMonitor.IfReturnHTMLResponse(w, r)
+	if htmlResponse {
+		indexPageHTML(w, r)
+	} else {
+		indexPageText(w, r)
+	}
+}
+
+func indexPageText(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte("Available HTTP endpoints:\n"))
 
 	spacing := 0
@@ -184,13 +193,35 @@ func indexPage(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 	spacing = spacing + 3
+	formatter := fmt.Sprintf("%%-%ds: %%s\n", spacing)
 
-	formattedString := fmt.Sprintf("%%-%ds: %%s\n", spacing)
 	for _, endpoint := range endpoints {
-		w.Write([]byte(fmt.Sprintf(formattedString, endpoint.path, endpoint.desc)))
+		w.Write([]byte(fmt.Sprintf(formatter, endpoint.path, endpoint.desc)))
 	}
 }
 
+func indexPageHTML(w http.ResponseWriter, r *http.Request) {
+
+	w.Write([]byte("<h1>Available HTTP endpoints:</h1>\n"))
+
+	var formattedString string
+	needLinkPaths := []string{"/metrics", "/sandboxes"}
+
+	w.Write([]byte("<ul>"))
+	for _, endpoint := range endpoints {
+		formattedString = fmt.Sprintf("<b>%s</b>: %s\n", endpoint.path, endpoint.desc)
+		for _, linkPath := range needLinkPaths {
+			if linkPath == endpoint.path {
+				formattedString = fmt.Sprintf("<b><a href='%s'>%s</a></b>: %s\n", endpoint.path, endpoint.path, endpoint.desc)
+				break
+			}
+		}
+		formattedString = fmt.Sprintf("<li>%s</li>", formattedString)
+		w.Write([]byte(formattedString))
+	}
+	w.Write([]byte("</ul>"))
+}
+
 // initLog setup logger
 func initLog() {
 	kataMonitorLog := logrus.WithFields(logrus.Fields{

src/runtime/config/configuration-clh.toml.in

@@ -125,7 +125,8 @@ virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
 #
 # Format example:
 #   ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"]
-#
+# Examples:
+#   Set virtiofsd log level to debug : ["-o", "log_level=debug"] or ["-d"]
 # see `virtiofsd -h` for possible options.
 virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@
 

src/runtime/config/configuration-qemu.toml.in

@@ -168,6 +168,8 @@ virtio_fs_cache_size = @DEFVIRTIOFSCACHESIZE@
 #
 # Format example:
 #   ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"]
+# Examples:
+#   Set virtiofsd log level to debug : ["-o", "log_level=debug"] or ["-d"]
 #
 # see `virtiofsd -h` for possible options.
 virtio_fs_extra_args = @DEFVIRTIOFSEXTRAARGS@

src/runtime/pkg/kata-monitor/metrics.go

@@ -78,6 +78,21 @@ func (km *KataMonitor) ProcessMetricsRequest(w http.ResponseWriter, r *http.Requ
 		scrapeDurationsHistogram.Observe(float64(time.Since(start).Nanoseconds() / int64(time.Millisecond)))
 	}()
 
+	// this is likely the same as `kata-runtime metrics <SANDBOX>`.
+	sandboxID, err := getSandboxIDFromReq(r)
+	if err == nil && sandboxID != "" {
+		metrics, err := GetSandboxMetrics(sandboxID)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(err.Error()))
+			return
+		}
+		w.Write([]byte(metrics))
+		return
+	}
+
+	// if no sandbox provided, will get all sandbox's metrics.
+
 	// prepare writer for writing response.
 	contentType := expfmt.Negotiate(r.Header)
 

src/runtime/pkg/kata-monitor/monitor.go

@@ -27,6 +27,7 @@ const (
 	RuntimeCRIO                 = "cri-o"
 	fsMonitorRetryDelaySeconds  = 60
 	podCacheRefreshDelaySeconds = 5
+	contentTypeHtml             = "text/html"
 )
 
 // SetLogger sets the logger for katamonitor package.
@@ -194,7 +195,41 @@ func (km *KataMonitor) GetAgentURL(w http.ResponseWriter, r *http.Request) {
 // ListSandboxes list all sandboxes running in Kata
 func (km *KataMonitor) ListSandboxes(w http.ResponseWriter, r *http.Request) {
 	sandboxes := km.sandboxCache.getSandboxList()
+	htmlResponse := IfReturnHTMLResponse(w, r)
+	if htmlResponse {
+		listSandboxesHtml(sandboxes, w)
+	} else {
+		listSandboxesText(sandboxes, w)
+	}
+}
+
+func listSandboxesText(sandboxes []string, w http.ResponseWriter) {
 	for _, s := range sandboxes {
 		w.Write([]byte(fmt.Sprintf("%s\n", s)))
 	}
 }
+func listSandboxesHtml(sandboxes []string, w http.ResponseWriter) {
+	w.Write([]byte("<h1>Sandbox list</h1>\n"))
+	w.Write([]byte("<ul>\n"))
+	for _, s := range sandboxes {
+		w.Write([]byte(fmt.Sprintf("<li>%s: <a href='/debug/pprof/?sandbox=%s'>pprof</a>, <a href='/metrics?sandbox=%s'>metrics</a>, <a href='/agent-url?sandbox=%s'>agent-url</a></li>\n", s, s, s, s)))
+	}
+	w.Write([]byte("</ul>\n"))
+}
+
+// IfReturnHTMLResponse returns true if request accepts html response
+// NOTE: IfReturnHTMLResponse will also set response header to `text/html`
+func IfReturnHTMLResponse(w http.ResponseWriter, r *http.Request) bool {
+	accepts := r.Header["Accept"]
+	for _, accept := range accepts {
+		fields := strings.Split(accept, ",")
+		for _, field := range fields {
+			if field == contentTypeHtml {
+				w.Header().Set("Content-Type", contentTypeHtml)
+				return true
+			}
+		}
+	}
+
+	return false
+}

src/runtime/pkg/kata-monitor/pprof.go

@@ -55,8 +55,10 @@ func (km *KataMonitor) proxyRequest(w http.ResponseWriter, r *http.Request) {
 	}
 
 	uri := fmt.Sprintf("http://shim%s", r.URL.String())
+	monitorLog.Debugf("proxyRequest to: %s, uri: %s", socketAddress, uri)
 	resp, err := client.Get(uri)
 	if err != nil {
+		serveError(w, http.StatusInternalServerError, fmt.Sprintf("failed to request %s through %s", uri, socketAddress))
 		return
 	}
 

src/runtime/virtcontainers/clh.go

@@ -234,7 +234,6 @@ func (clh *cloudHypervisor) createVirtiofsDaemon(sharedPath string) (VirtiofsDae
 		sourcePath: sharedPath,
 		socketPath: virtiofsdSocketPath,
 		extraArgs:  clh.config.VirtioFSExtraArgs,
-		debug:      clh.config.Debug,
 		cache:      clh.config.VirtioFSCache,
 	}, nil
 }
@@ -302,7 +301,6 @@ func (clh *cloudHypervisor) loadVirtiofsDaemon(sharedPath string) (VirtiofsDaemo
 	return &virtiofsd{
 		PID:        clh.state.VirtiofsDaemonPid,
 		sourcePath: sharedPath,
-		debug:      clh.config.Debug,
 		socketPath: virtiofsdSocketPath,
 	}, nil
 }

src/runtime/virtcontainers/qemu.go

@@ -499,7 +499,6 @@ func (q *qemu) createVirtiofsDaemon(sharedPath string) (VirtiofsDaemon, error) {
 		sourcePath: sharedPath,
 		socketPath: virtiofsdSocketPath,
 		extraArgs:  q.config.VirtioFSExtraArgs,
-		debug:      q.config.Debug,
 		cache:      q.config.VirtioFSCache,
 	}, nil
 }

src/runtime/virtcontainers/virtiofsd.go

@@ -70,8 +70,6 @@ type virtiofsd struct {
 	sourcePath string
 	// extraArgs list of extra args to append to virtiofsd command
 	extraArgs []string
-	// debug flag
-	debug bool
 	// PID process ID of virtiosd process
 	PID int
 }
@@ -199,14 +197,8 @@ func (v *virtiofsd) args(FdSocketNumber uint) ([]string, error) {
 		"-o", "source=" + v.sourcePath,
 		// fd number of vhost-user socket
 		fmt.Sprintf("--fd=%v", FdSocketNumber),
-	}
-
-	if v.debug {
-		// enable debug output (implies -f)
-		args = append(args, "-d")
-	} else {
 		// foreground operation
-		args = append(args, "-f")
+		"-f",
 	}
 
 	if len(v.extraArgs) != 0 {

src/runtime/virtcontainers/virtiofsd_test.go

@@ -22,7 +22,6 @@ func TestVirtiofsdStart(t *testing.T) {
 		cache      string
 		extraArgs  []string
 		sourcePath string
-		debug      bool
 		PID        int
 		ctx        context.Context
 	}
@@ -58,7 +57,6 @@ func TestVirtiofsdStart(t *testing.T) {
 				cache:      tt.fields.cache,
 				extraArgs:  tt.fields.extraArgs,
 				sourcePath: tt.fields.sourcePath,
-				debug:      tt.fields.debug,
 				PID:        tt.fields.PID,
 				ctx:        tt.fields.ctx,
 			}
@@ -86,7 +84,6 @@ func TestVirtiofsdArgs(t *testing.T) {
 	assert.NoError(err)
 	assert.Equal(expected, strings.Join(args, " "))
 
-	v.debug = false
 	expected = "--syslog -o cache=none -o no_posix_lock -o source=/run/kata-shared/foo --fd=456 -f"
 	args, err = v.args(456)
 	assert.NoError(err)