diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index fa6320fde4..dda460badb 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -83,6 +83,15 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.2.1" @@ -538,6 +547,7 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "ttrpc", + "vsock-exporter", ] [[package]] @@ -843,6 +853,7 @@ dependencies = [ "percent-encoding", "pin-project", "rand", + "serde", "thiserror", ] @@ -1301,6 +1312,9 @@ name = "serde" version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" +dependencies = [ + "serde_derive", +] [[package]] name = "serde_derive" @@ -1756,6 +1770,22 @@ dependencies = [ "nix 0.19.1", ] +[[package]] +name = "vsock-exporter" +version = "0.1.0" +dependencies = [ + "async-trait", + "bincode", + "byteorder", + "libc", + "nix 0.20.0", + "opentelemetry", + "serde", + "slog", + "thiserror", + "vsock", +] + [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 98aecf7917..a6dc73fe54 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -50,6 +50,7 @@ tracing = "0.1.26" tracing-subscriber = "0.2.18" tracing-opentelemetry = "0.13.0" opentelemetry = "0.14.0" +vsock-exporter = { path = "vsock-exporter" } [dev-dependencies] tempfile = "3.1.0" diff --git a/src/agent/src/config.rs b/src/agent/src/config.rs index 2bb70dfb80..01ae7a226a 100644 --- a/src/agent/src/config.rs +++ b/src/agent/src/config.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 // +use crate::tracer; use anyhow::{anyhow, Result}; use std::env; use std::fs; @@ -10,6 +11,7 @@ use tracing::instrument; const DEBUG_CONSOLE_FLAG: &str = "agent.debug_console"; const DEV_MODE_FLAG: &str = "agent.devmode"; +const TRACE_MODE_OPTION: &str = "agent.trace"; const LOG_LEVEL_OPTION: &str = "agent.log"; const SERVER_ADDR_OPTION: &str = "agent.server_addr"; const HOTPLUG_TIMOUT_OPTION: &str = "agent.hotplug_timeout"; @@ -27,6 +29,7 @@ const VSOCK_PORT: u16 = 1024; // Environment variables used for development and testing const SERVER_ADDR_ENV_VAR: &str = "KATA_AGENT_SERVER_ADDR"; const LOG_LEVEL_ENV_VAR: &str = "KATA_AGENT_LOG_LEVEL"; +const TRACE_TYPE_ENV_VAR: &str = "KATA_AGENT_TRACE_TYPE"; const ERR_INVALID_LOG_LEVEL: &str = "invalid log level"; const ERR_INVALID_LOG_LEVEL_PARAM: &str = "invalid log level parameter"; @@ -55,6 +58,7 @@ pub struct AgentConfig { pub container_pipe_size: i32, pub server_addr: String, pub unified_cgroup_hierarchy: bool, + pub tracing: tracer::TraceType, } // parse_cmdline_param parse commandline parameters. @@ -99,6 +103,7 @@ impl AgentConfig { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: format!("{}:{}", VSOCK_ADDR, VSOCK_PORT), unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, } } @@ -111,6 +116,15 @@ impl AgentConfig { parse_cmdline_param!(param, DEBUG_CONSOLE_FLAG, self.debug_console); parse_cmdline_param!(param, DEV_MODE_FLAG, self.dev_mode); + // Support "bare" tracing option for backwards compatibility with + // Kata 1.x. + if param == &TRACE_MODE_OPTION { + self.tracing = tracer::TraceType::Isolated; + continue; + } + + parse_cmdline_param!(param, TRACE_MODE_OPTION, self.tracing, get_trace_type); + // parse cmdline options parse_cmdline_param!(param, LOG_LEVEL_OPTION, self.log_level, get_log_level); parse_cmdline_param!( @@ -169,6 +183,12 @@ impl AgentConfig { } } + if let Ok(value) = env::var(TRACE_TYPE_ENV_VAR) { + if let Ok(result) = value.parse::() { + self.tracing = result; + } + } + Ok(()) } } @@ -226,6 +246,27 @@ fn get_log_level(param: &str) -> Result { } } +#[instrument] +fn get_trace_type(param: &str) -> Result { + if param.is_empty() { + return Err(anyhow!("invalid trace type parameter")); + } + + let fields: Vec<&str> = param.split('=').collect(); + + if fields[0] != TRACE_MODE_OPTION { + return Err(anyhow!("invalid trace type key name")); + } + + if fields.len() == 1 { + return Ok(tracer::TraceType::Isolated); + } + + let result = fields[1].parse::()?; + + Ok(result) +} + #[instrument] fn get_hotplug_timeout(param: &str) -> Result { let fields: Vec<&str> = param.split('=').collect(); @@ -328,6 +369,10 @@ mod tests { use std::time; use tempfile::tempdir; + const ERR_INVALID_TRACE_TYPE_PARAM: &str = "invalid trace type parameter"; + const ERR_INVALID_TRACE_TYPE: &str = "invalid trace type"; + const ERR_INVALID_TRACE_TYPE_KEY: &str = "invalid trace type key name"; + // helper function to make errors less crazy-long fn make_err(desc: &str) -> Error { anyhow!(desc.to_string()) @@ -383,6 +428,7 @@ mod tests { container_pipe_size: i32, server_addr: &'a str, unified_cgroup_hierarchy: bool, + tracing: tracer::TraceType, } let tests = &[ @@ -396,6 +442,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.debug_console agent.devmodex", @@ -407,6 +454,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.logx=debug", @@ -418,6 +466,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.log=debug", @@ -429,6 +478,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.log=debug", @@ -440,6 +490,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "", @@ -451,6 +502,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "foo", @@ -462,6 +514,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "foo bar", @@ -473,6 +526,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "foo bar", @@ -484,6 +538,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "foo agent bar", @@ -495,6 +550,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "foo debug_console agent bar devmode", @@ -506,6 +562,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.debug_console", @@ -517,6 +574,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: " agent.debug_console ", @@ -528,6 +586,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.debug_console foo", @@ -539,6 +598,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: " agent.debug_console foo", @@ -550,6 +610,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "foo agent.debug_console bar", @@ -561,6 +622,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "foo agent.debug_console", @@ -572,6 +634,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "foo agent.debug_console ", @@ -583,6 +646,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.devmode", @@ -594,6 +658,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: " agent.devmode ", @@ -605,6 +670,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.devmode foo", @@ -616,6 +682,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: " agent.devmode foo", @@ -627,6 +694,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "foo agent.devmode bar", @@ -638,6 +706,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "foo agent.devmode", @@ -649,6 +718,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "foo agent.devmode ", @@ -660,6 +730,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.devmode agent.debug_console", @@ -671,6 +742,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.devmode agent.debug_console agent.hotplug_timeout=100 agent.unified_cgroup_hierarchy=a", @@ -682,6 +754,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.devmode agent.debug_console agent.hotplug_timeout=0 agent.unified_cgroup_hierarchy=11", @@ -693,6 +766,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: true, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.devmode agent.debug_console agent.container_pipe_size=2097152 agent.unified_cgroup_hierarchy=false", @@ -704,6 +778,7 @@ mod tests { container_pipe_size: 2097152, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.devmode agent.debug_console agent.container_pipe_size=100 agent.unified_cgroup_hierarchy=true", @@ -715,6 +790,7 @@ mod tests { container_pipe_size: 100, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: true, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.devmode agent.debug_console agent.container_pipe_size=0 agent.unified_cgroup_hierarchy=0", @@ -726,6 +802,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.devmode agent.debug_console agent.container_pip_siz=100 agent.unified_cgroup_hierarchy=1", @@ -737,6 +814,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: true, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "", @@ -748,6 +826,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "", @@ -759,6 +838,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: "foo", unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "", @@ -770,6 +850,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: "=", unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "", @@ -781,6 +862,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: "=foo", unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "", @@ -792,6 +874,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: "foo=bar=baz=", unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "", @@ -803,6 +886,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: "unix:///tmp/foo.socket", unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "", @@ -814,6 +898,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: "unix://@/tmp/foo.socket", unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "", @@ -825,6 +910,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "", @@ -836,6 +922,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "", @@ -847,6 +934,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "", @@ -858,6 +946,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "server_addr=unix:///tmp/foo.socket", @@ -869,6 +958,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.server_address=unix:///tmp/foo.socket", @@ -880,6 +970,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: TEST_SERVER_ADDR, unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: "agent.server_addr=unix:///tmp/foo.socket", @@ -891,6 +982,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: "unix:///tmp/foo.socket", unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: " agent.server_addr=unix:///tmp/foo.socket", @@ -902,6 +994,7 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: "unix:///tmp/foo.socket", unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, TestData { contents: " agent.server_addr=unix:///tmp/foo.socket a", @@ -913,6 +1006,115 @@ mod tests { container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, server_addr: "unix:///tmp/foo.socket", unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, + }, + TestData { + contents: "trace", + env_vars: Vec::new(), + debug_console: false, + dev_mode: false, + log_level: DEFAULT_LOG_LEVEL, + hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT, + container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, + server_addr: TEST_SERVER_ADDR, + unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, + }, + TestData { + contents: ".trace", + env_vars: Vec::new(), + debug_console: false, + dev_mode: false, + log_level: DEFAULT_LOG_LEVEL, + hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT, + container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, + server_addr: TEST_SERVER_ADDR, + unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, + }, + TestData { + contents: "agent.tracer", + env_vars: Vec::new(), + debug_console: false, + dev_mode: false, + log_level: DEFAULT_LOG_LEVEL, + hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT, + container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, + server_addr: TEST_SERVER_ADDR, + unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, + }, + TestData { + contents: "agent.trac", + env_vars: Vec::new(), + debug_console: false, + dev_mode: false, + log_level: DEFAULT_LOG_LEVEL, + hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT, + container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, + server_addr: TEST_SERVER_ADDR, + unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, + }, + TestData { + contents: "agent.trace", + env_vars: Vec::new(), + debug_console: false, + dev_mode: false, + log_level: DEFAULT_LOG_LEVEL, + hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT, + container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, + server_addr: TEST_SERVER_ADDR, + unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Isolated, + }, + TestData { + contents: "agent.trace=isolated", + env_vars: Vec::new(), + debug_console: false, + dev_mode: false, + log_level: DEFAULT_LOG_LEVEL, + hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT, + container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, + server_addr: TEST_SERVER_ADDR, + unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Isolated, + }, + TestData { + contents: "agent.trace=disabled", + env_vars: Vec::new(), + debug_console: false, + dev_mode: false, + log_level: DEFAULT_LOG_LEVEL, + hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT, + container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, + server_addr: TEST_SERVER_ADDR, + unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, + }, + TestData { + contents: "", + env_vars: vec!["KATA_AGENT_TRACE_TYPE=isolated"], + debug_console: false, + dev_mode: false, + log_level: DEFAULT_LOG_LEVEL, + hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT, + container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, + server_addr: TEST_SERVER_ADDR, + unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Isolated, + }, + TestData { + contents: "", + env_vars: vec!["KATA_AGENT_TRACE_TYPE=disabled"], + debug_console: false, + dev_mode: false, + log_level: DEFAULT_LOG_LEVEL, + hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT, + container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE, + server_addr: TEST_SERVER_ADDR, + unified_cgroup_hierarchy: false, + tracing: tracer::TraceType::Disabled, }, ]; @@ -967,6 +1169,7 @@ mod tests { ); assert_eq!(config.container_pipe_size, 0, "{}", msg); assert_eq!(config.server_addr, TEST_SERVER_ADDR, "{}", msg); + assert_eq!(config.tracing, tracer::TraceType::Disabled, "{}", msg); let result = config.parse_cmdline(filename); assert!(result.is_ok(), "{}", msg); @@ -982,6 +1185,7 @@ mod tests { assert_eq!(d.hotplug_timeout, config.hotplug_timeout, "{}", msg); assert_eq!(d.container_pipe_size, config.container_pipe_size, "{}", msg); assert_eq!(d.server_addr, config.server_addr, "{}", msg); + assert_eq!(d.tracing, config.tracing, "{}", msg); for v in vars_to_unset { env::remove_var(v); @@ -1378,4 +1582,62 @@ mod tests { assert_result!(d.result, result, msg); } } + + #[test] + fn test_get_trace_type() { + #[derive(Debug)] + struct TestData<'a> { + param: &'a str, + result: Result, + } + + let tests = &[ + TestData { + param: "", + result: Err(make_err(ERR_INVALID_TRACE_TYPE_PARAM)), + }, + TestData { + param: "agent.tracer", + result: Err(make_err(ERR_INVALID_TRACE_TYPE_KEY)), + }, + TestData { + param: "agent.trac", + result: Err(make_err(ERR_INVALID_TRACE_TYPE_KEY)), + }, + TestData { + param: "agent.trace=", + result: Err(make_err(ERR_INVALID_TRACE_TYPE)), + }, + TestData { + param: "agent.trace==", + result: Err(make_err(ERR_INVALID_TRACE_TYPE)), + }, + TestData { + param: "agent.trace=foo", + result: Err(make_err(ERR_INVALID_TRACE_TYPE)), + }, + TestData { + param: "agent.trace", + result: Ok(tracer::TraceType::Isolated), + }, + TestData { + param: "agent.trace=isolated", + result: Ok(tracer::TraceType::Isolated), + }, + TestData { + param: "agent.trace=disabled", + result: Ok(tracer::TraceType::Disabled), + }, + ]; + + for (i, d) in tests.iter().enumerate() { + let msg = format!("test[{}]: {:?}", i, d); + + let result = get_trace_type(d.param); + + let msg = format!("{}: result: {:?}", msg, result); + + assert_result!(d.result, result, msg); + } + } } diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 4431158355..b51cbb47ee 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -32,7 +32,7 @@ use std::os::unix::io::AsRawFd; use std::path::Path; use std::process::exit; use std::sync::Arc; -use tracing::instrument; +use tracing::{instrument, span}; mod config; mod console; @@ -56,7 +56,7 @@ mod version; use mount::{cgroups_mount, general_mount}; use sandbox::Sandbox; use signal::setup_signal_handler; -use slog::Logger; +use slog::{error, info, o, warn, Logger}; use uevent::watch_uevents; use futures::future::join_all; @@ -71,6 +71,7 @@ use tokio::{ }; mod rpc; +mod tracer; const NAME: &str = "kata-agent"; const KERNEL_CMDLINE_FILE: &str = "/proc/cmdline"; @@ -201,6 +202,17 @@ async fn real_main() -> std::result::Result<(), Box> { ttrpc_log_guard = Ok(slog_stdlog::init().map_err(|e| e)?); } + if config.tracing != tracer::TraceType::Disabled { + let _ = tracer::setup_tracing(NAME, &logger, &config)?; + } + + let root = span!(tracing::Level::TRACE, "root-span", work_units = 2); + + // XXX: Start the root trace transaction. + // + // XXX: Note that *ALL* spans needs to start after this point!! + let _enter = root.enter(); + // Start the sandbox and wait for its ttRPC server to end start_sandbox(&logger, &config, init_mode, &mut tasks, shutdown_rx.clone()).await?; @@ -229,6 +241,10 @@ async fn real_main() -> std::result::Result<(), Box> { } } + if config.tracing != tracer::TraceType::Disabled { + tracer::end_tracing(); + } + eprintln!("{} shutdown complete", NAME); Ok(()) diff --git a/src/agent/src/tracer.rs b/src/agent/src/tracer.rs new file mode 100644 index 0000000000..4ae5111eb4 --- /dev/null +++ b/src/agent/src/tracer.rs @@ -0,0 +1,91 @@ +// Copyright (c) 2020-2021 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +use crate::config::AgentConfig; +use anyhow::Result; +use opentelemetry::{global, sdk::trace::Config, trace::TracerProvider}; +use slog::{info, o, Logger}; +use std::error::Error; +use std::fmt; +use std::str::FromStr; +use tracing_opentelemetry::OpenTelemetryLayer; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::Registry; + +#[derive(Debug, PartialEq)] +pub enum TraceType { + Disabled, + Isolated, +} + +#[derive(Debug)] +pub struct TraceTypeError { + details: String, +} + +impl TraceTypeError { + fn new(msg: &str) -> TraceTypeError { + TraceTypeError { + details: msg.into(), + } + } +} + +impl Error for TraceTypeError {} + +impl fmt::Display for TraceTypeError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.details) + } +} + +impl FromStr for TraceType { + type Err = TraceTypeError; + + fn from_str(s: &str) -> Result { + match s { + "isolated" => Ok(TraceType::Isolated), + "disabled" => Ok(TraceType::Disabled), + _ => Err(TraceTypeError::new("invalid trace type")), + } + } +} + +pub fn setup_tracing(name: &'static str, logger: &Logger, _agent_cfg: &AgentConfig) -> Result<()> { + let logger = logger.new(o!("subsystem" => "vsock-tracer")); + + let exporter = vsock_exporter::Exporter::builder() + .with_logger(&logger) + .init(); + + let config = Config::default(); + + let builder = opentelemetry::sdk::trace::TracerProvider::builder() + .with_simple_exporter(exporter) + .with_config(config); + + let provider = builder.build(); + + // We don't need a versioned tracer. + let version = None; + + let tracer = provider.get_tracer(name, version); + + let _global_provider = global::set_tracer_provider(provider); + + let layer = OpenTelemetryLayer::new(tracer); + + let subscriber = Registry::default().with(layer); + + tracing::subscriber::set_global_default(subscriber)?; + + info!(logger, "tracing setup"); + + Ok(()) +} + +pub fn end_tracing() { + global::shutdown_tracer_provider(); +} diff --git a/src/agent/vsock-exporter/Cargo.toml b/src/agent/vsock-exporter/Cargo.toml new file mode 100644 index 0000000000..b91ca4f18c --- /dev/null +++ b/src/agent/vsock-exporter/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "vsock-exporter" +version = "0.1.0" +authors = ["James O. D. Hunt "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +nix = "0.20.0" +libc = "0.2.94" +thiserror = "1.0.24" +opentelemetry = { version = "0.14.0", features=["serialize"] } +serde = { version = "1.0.126", features = ["derive"] } +vsock = "0.2.3" +bincode = "1.3.3" +byteorder = "1.4.3" +slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"] } +async-trait = "0.1.50" diff --git a/src/agent/vsock-exporter/src/lib.rs b/src/agent/vsock-exporter/src/lib.rs new file mode 100644 index 0000000000..99aaf341bb --- /dev/null +++ b/src/agent/vsock-exporter/src/lib.rs @@ -0,0 +1,196 @@ +// Copyright (c) 2020-2021 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +// The VSOCK Exporter sends trace spans "out" to the forwarder running on the +// host (which then forwards them on to a trace collector). The data is sent +// via a VSOCK socket that the forwarder process is listening on. To allow the +// forwarder to know how much data to each for each trace span the simplest +// protocol is employed which uses a header packet and the payload (trace +// span) data. The header packet is a simple count of the number of bytes in the +// payload, which allows the forwarder to know how many bytes it must read to +// consume the trace span. The payload is a serialised version of the trace span. + +use async_trait::async_trait; +use byteorder::{ByteOrder, NetworkEndian}; +use opentelemetry::sdk::export::trace::{ExportResult, SpanData, SpanExporter}; +use opentelemetry::sdk::export::ExportError; +use slog::{error, o, Logger}; +use std::io::{ErrorKind, Write}; +use std::net::Shutdown; +use std::sync::Mutex; +use vsock::{SockAddr, VsockStream}; + +const ANY_CID: &str = "any"; + +// Must match the value of the variable of the same name in the trace forwarder. +const HEADER_SIZE_BYTES: u64 = std::mem::size_of::() as u64; + +// By default, the VSOCK exporter should talk "out" to the host where the +// forwarder is running. +const DEFAULT_CID: u32 = libc::VMADDR_CID_HOST; + +// The VSOCK port the forwarders listens on by default +const DEFAULT_PORT: u32 = 10240; + +#[derive(Debug)] +pub struct Exporter { + port: u32, + cid: u32, + conn: Mutex, + logger: Logger, +} + +impl Exporter { + /// Create a new exporter builder. + pub fn builder() -> Builder { + Builder::default() + } +} + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("connection error: {0}")] + ConnectionError(String), + #[error("serialisation error: {0}")] + SerialisationError(#[from] bincode::Error), + #[error("I/O error: {0}")] + IOError(#[from] std::io::Error), +} + +impl ExportError for Error { + fn exporter_name(&self) -> &'static str { + "vsock-exporter" + } +} + +fn make_io_error(desc: String) -> std::io::Error { + std::io::Error::new(ErrorKind::Other, desc) +} + +// Send a trace span to the forwarder running on the host. +fn write_span(writer: &mut dyn Write, span: &SpanData) -> Result<(), std::io::Error> { + let encoded_payload: Vec = + bincode::serialize(&span).map_err(|e| make_io_error(e.to_string()))?; + + let payload_len: u64 = encoded_payload.len() as u64; + + let mut payload_len_as_bytes: [u8; HEADER_SIZE_BYTES as usize] = + [0; HEADER_SIZE_BYTES as usize]; + + // Encode the header + NetworkEndian::write_u64(&mut payload_len_as_bytes, payload_len); + + // Send the header + writer + .write_all(&payload_len_as_bytes) + .map_err(|e| make_io_error(format!("failed to write trace header: {:?}", e)))?; + + writer + .write_all(&encoded_payload) + .map_err(|e| make_io_error(format!("failed to write trace payload: {:?}", e))) +} + +fn handle_batch(writer: &mut dyn Write, batch: Vec) -> ExportResult { + for span_data in batch { + write_span(writer, &span_data).map_err(Error::IOError)?; + } + + Ok(()) +} + +#[async_trait] +impl SpanExporter for Exporter { + async fn export(&mut self, batch: Vec) -> ExportResult { + let conn = self.conn.lock(); + + match conn { + Ok(mut c) => handle_batch(&mut *c, batch), + Err(e) => { + error!(self.logger, "failed to obtain connection"; + "error" => format!("{}", e)); + + return Err(Error::ConnectionError(e.to_string()).into()); + } + } + } + + fn shutdown(&mut self) { + let conn = match self.conn.lock() { + Ok(conn) => conn, + Err(e) => { + error!(self.logger, "failed to obtain connection"; + "error" => format!("{}", e)); + return; + } + }; + + conn.shutdown(Shutdown::Write) + .expect("failed to shutdown VSOCK connection"); + } +} + +#[derive(Debug)] +pub struct Builder { + port: u32, + cid: u32, + logger: Logger, +} + +impl Default for Builder { + fn default() -> Self { + let logger = Logger::root(slog::Discard, o!()); + + Builder { + cid: DEFAULT_CID, + port: DEFAULT_PORT, + logger, + } + } +} + +impl Builder { + pub fn with_cid(self, cid: u32) -> Self { + Builder { cid, ..self } + } + + pub fn with_port(self, port: u32) -> Self { + Builder { port, ..self } + } + + pub fn with_logger(self, logger: &Logger) -> Self { + Builder { + logger: logger.new(o!()), + ..self + } + } + + pub fn init(self) -> Exporter { + let Builder { port, cid, logger } = self; + + let sock_addr = SockAddr::new_vsock(self.cid, self.port); + + let cid_str: String; + + if self.cid == libc::VMADDR_CID_ANY { + cid_str = ANY_CID.to_string(); + } else { + cid_str = format!("{}", self.cid); + } + + let msg = format!( + "failed to connect to VSOCK server (port: {}, cid: {}) - {}", + self.port, cid_str, "ensure trace forwarder is running on host" + ); + + let conn = VsockStream::connect(&sock_addr).expect(&msg); + + Exporter { + port, + cid, + conn: Mutex::new(conn), + logger: logger.new(o!("cid" => cid_str, "port" => port)), + } + } +} diff --git a/src/trace-forwarder/Cargo.lock b/src/trace-forwarder/Cargo.lock index b84b54e7fe..b6d4c1beac 100644 --- a/src/trace-forwarder/Cargo.lock +++ b/src/trace-forwarder/Cargo.lock @@ -699,18 +699,18 @@ checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" [[package]] name = "serde" -version = "1.0.114" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3" +checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.114" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e" +checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43" dependencies = [ "proc-macro2", "quote", diff --git a/src/trace-forwarder/Cargo.toml b/src/trace-forwarder/Cargo.toml index 9dccdd86ba..b2344a51e6 100644 --- a/src/trace-forwarder/Cargo.toml +++ b/src/trace-forwarder/Cargo.toml @@ -15,7 +15,7 @@ clap = "2.33.0" vsock = "0.2.3" nix = "0.20.0" libc = "0.2.94" -serde = { version = "1.0.106", features = ["derive"] } +serde = { version = "1.0.126", features = ["derive"] } bincode = "1.3.3" byteorder = "1.4.3" serde_json = "1.0.44" diff --git a/src/trace-forwarder/src/handler.rs b/src/trace-forwarder/src/handler.rs index ac53b0ca2c..377ee0e82d 100644 --- a/src/trace-forwarder/src/handler.rs +++ b/src/trace-forwarder/src/handler.rs @@ -19,6 +19,9 @@ use vsock::VsockStream; // This constant defines the number of bytes used to encode the header on the // wire. In other words, the first 64-bits of the packet contain a number // specifying how many bytes are in the remainder of the packet. +// +// Must match the value of the variable of the same name in the agents +// vsock-exporter. const HEADER_SIZE_BYTES: u64 = std::mem::size_of::() as u64; fn mk_io_err(msg: &str) -> std::io::Error {