github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-05-01 05:04:26 +00:00
Code Issues Projects Releases Wiki Activity

tracing: Add basic VSOCK tracing

Browse Source 
Implement an openTelemetry custom exporter that sends trace spans to a
VSOCK socket. A VSOCK-to-span converter (such as the Kata trace
forwarder) needs to be running on the host to allow systems like Jaeger
to capture the trace spans.

By default, tracing is not enabled (meaning a NOP tracer is used). To
activate tracing, set the `agent.kata.enable_tracing=true` in the
configuration file.

The type of tracing this change introduces is "static isolated"
tracing. See [1] for further details.

> **Note:**
>
> This change only provides the foundational changes for agent
> tracing work. The feature is _not_ yet complete since it does
> not yet show the correct trace hierarchy.

Fixes: #60.

[1] - https://github.com/kata-containers/agent/blob/master/TRACING.md

Signed-off-by: James O. D. Hunt <james.o.hunt@intel.com>
This commit is contained in:
James O. D. Hunt 2021-05-26 11:49:19 +01:00
parent b25ad1ab2c
commit a9a0eccf33
10 changed files with 625 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/agent/Cargo.lock generated
View File

@ -83,6 +83,15 @@ dependencies = [
"rustc-demangle",
]
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde",
]
[[package]]
name = "bitflags"
version = "1.2.1"
@ -538,6 +547,7 @@ dependencies = [
"tracing-opentelemetry",
"tracing-subscriber",
"ttrpc",
"vsock-exporter",
]
[[package]]
@ -843,6 +853,7 @@ dependencies = [
"percent-encoding",
"pin-project",
"rand",
"serde",
"thiserror",
]
@ -1301,6 +1312,9 @@ name = "serde"
version = "1.0.126"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
@ -1756,6 +1770,22 @@ dependencies = [
"nix 0.19.1",
]
[[package]]
name = "vsock-exporter"
version = "0.1.0"
dependencies = [
"async-trait",
"bincode",
"byteorder",
"libc",
"nix 0.20.0",
"opentelemetry",
"serde",
"slog",
"thiserror",
"vsock",
]
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"

1
src/agent/Cargo.toml
View File

@ -50,6 +50,7 @@ tracing = "0.1.26"
tracing-subscriber = "0.2.18"
tracing-opentelemetry = "0.13.0"
opentelemetry = "0.14.0"
vsock-exporter = { path = "vsock-exporter" }
[dev-dependencies]
tempfile = "3.1.0"

262
src/agent/src/config.rs
View File

@ -2,6 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0
//
use crate::tracer;
use anyhow::{anyhow, Result};
use std::env;
use std::fs;
@ -10,6 +11,7 @@ use tracing::instrument;
const DEBUG_CONSOLE_FLAG: &str = "agent.debug_console";
const DEV_MODE_FLAG: &str = "agent.devmode";
const TRACE_MODE_OPTION: &str = "agent.trace";
const LOG_LEVEL_OPTION: &str = "agent.log";
const SERVER_ADDR_OPTION: &str = "agent.server_addr";
const HOTPLUG_TIMOUT_OPTION: &str = "agent.hotplug_timeout";
@ -27,6 +29,7 @@ const VSOCK_PORT: u16 = 1024;
// Environment variables used for development and testing
const SERVER_ADDR_ENV_VAR: &str = "KATA_AGENT_SERVER_ADDR";
const LOG_LEVEL_ENV_VAR: &str = "KATA_AGENT_LOG_LEVEL";
const TRACE_TYPE_ENV_VAR: &str = "KATA_AGENT_TRACE_TYPE";
const ERR_INVALID_LOG_LEVEL: &str = "invalid log level";
const ERR_INVALID_LOG_LEVEL_PARAM: &str = "invalid log level parameter";
@ -55,6 +58,7 @@ pub struct AgentConfig {
pub container_pipe_size: i32,
pub server_addr: String,
pub unified_cgroup_hierarchy: bool,
pub tracing: tracer::TraceType,
}
// parse_cmdline_param parse commandline parameters.
@ -99,6 +103,7 @@ impl AgentConfig {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: format!("{}:{}", VSOCK_ADDR, VSOCK_PORT),
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
}
}
@ -111,6 +116,15 @@ impl AgentConfig {
parse_cmdline_param!(param, DEBUG_CONSOLE_FLAG, self.debug_console);
parse_cmdline_param!(param, DEV_MODE_FLAG, self.dev_mode);
// Support "bare" tracing option for backwards compatibility with
// Kata 1.x.
if param == &TRACE_MODE_OPTION {
self.tracing = tracer::TraceType::Isolated;
continue;
}
parse_cmdline_param!(param, TRACE_MODE_OPTION, self.tracing, get_trace_type);
// parse cmdline options
parse_cmdline_param!(param, LOG_LEVEL_OPTION, self.log_level, get_log_level);
parse_cmdline_param!(
@ -169,6 +183,12 @@ impl AgentConfig {
}
}
if let Ok(value) = env::var(TRACE_TYPE_ENV_VAR) {
if let Ok(result) = value.parse::<tracer::TraceType>() {
self.tracing = result;
}
}
Ok(())
}
}
@ -226,6 +246,27 @@ fn get_log_level(param: &str) -> Result<slog::Level> {
}
}
#[instrument]
fn get_trace_type(param: &str) -> Result<tracer::TraceType> {
if param.is_empty() {
return Err(anyhow!("invalid trace type parameter"));
}
let fields: Vec<&str> = param.split('=').collect();
if fields[0] != TRACE_MODE_OPTION {
return Err(anyhow!("invalid trace type key name"));
}
if fields.len() == 1 {
return Ok(tracer::TraceType::Isolated);
}
let result = fields[1].parse::<tracer::TraceType>()?;
Ok(result)
}
#[instrument]
fn get_hotplug_timeout(param: &str) -> Result<time::Duration> {
let fields: Vec<&str> = param.split('=').collect();
@ -328,6 +369,10 @@ mod tests {
use std::time;
use tempfile::tempdir;
const ERR_INVALID_TRACE_TYPE_PARAM: &str = "invalid trace type parameter";
const ERR_INVALID_TRACE_TYPE: &str = "invalid trace type";
const ERR_INVALID_TRACE_TYPE_KEY: &str = "invalid trace type key name";
// helper function to make errors less crazy-long
fn make_err(desc: &str) -> Error {
anyhow!(desc.to_string())
@ -383,6 +428,7 @@ mod tests {
container_pipe_size: i32,
server_addr: &'a str,
unified_cgroup_hierarchy: bool,
tracing: tracer::TraceType,
}
let tests = &[
@ -396,6 +442,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.debug_console agent.devmodex",
@ -407,6 +454,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.logx=debug",
@ -418,6 +466,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.log=debug",
@ -429,6 +478,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.log=debug",
@ -440,6 +490,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "",
@ -451,6 +502,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "foo",
@ -462,6 +514,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "foo bar",
@ -473,6 +526,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "foo bar",
@ -484,6 +538,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "foo agent bar",
@ -495,6 +550,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "foo debug_console agent bar devmode",
@ -506,6 +562,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.debug_console",
@ -517,6 +574,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: " agent.debug_console ",
@ -528,6 +586,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.debug_console foo",
@ -539,6 +598,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: " agent.debug_console foo",
@ -550,6 +610,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "foo agent.debug_console bar",
@ -561,6 +622,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "foo agent.debug_console",
@ -572,6 +634,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "foo agent.debug_console ",
@ -583,6 +646,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.devmode",
@ -594,6 +658,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: " agent.devmode ",
@ -605,6 +670,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.devmode foo",
@ -616,6 +682,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: " agent.devmode foo",
@ -627,6 +694,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "foo agent.devmode bar",
@ -638,6 +706,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "foo agent.devmode",
@ -649,6 +718,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "foo agent.devmode ",
@ -660,6 +730,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.devmode agent.debug_console",
@ -671,6 +742,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.devmode agent.debug_console agent.hotplug_timeout=100 agent.unified_cgroup_hierarchy=a",
@ -682,6 +754,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.devmode agent.debug_console agent.hotplug_timeout=0 agent.unified_cgroup_hierarchy=11",
@ -693,6 +766,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: true,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.devmode agent.debug_console agent.container_pipe_size=2097152 agent.unified_cgroup_hierarchy=false",
@ -704,6 +778,7 @@ mod tests {
container_pipe_size: 2097152,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.devmode agent.debug_console agent.container_pipe_size=100 agent.unified_cgroup_hierarchy=true",
@ -715,6 +790,7 @@ mod tests {
container_pipe_size: 100,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: true,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.devmode agent.debug_console agent.container_pipe_size=0 agent.unified_cgroup_hierarchy=0",
@ -726,6 +802,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.devmode agent.debug_console agent.container_pip_siz=100 agent.unified_cgroup_hierarchy=1",
@ -737,6 +814,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: true,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "",
@ -748,6 +826,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "",
@ -759,6 +838,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: "foo",
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "",
@ -770,6 +850,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: "=",
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "",
@ -781,6 +862,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: "=foo",
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "",
@ -792,6 +874,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: "foo=bar=baz=",
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "",
@ -803,6 +886,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: "unix:///tmp/foo.socket",
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "",
@ -814,6 +898,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: "unix://@/tmp/foo.socket",
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "",
@ -825,6 +910,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "",
@ -836,6 +922,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "",
@ -847,6 +934,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "",
@ -858,6 +946,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "server_addr=unix:///tmp/foo.socket",
@ -869,6 +958,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.server_address=unix:///tmp/foo.socket",
@ -880,6 +970,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.server_addr=unix:///tmp/foo.socket",
@ -891,6 +982,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: "unix:///tmp/foo.socket",
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: " agent.server_addr=unix:///tmp/foo.socket",
@ -902,6 +994,7 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: "unix:///tmp/foo.socket",
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: " agent.server_addr=unix:///tmp/foo.socket a",
@ -913,6 +1006,115 @@ mod tests {
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: "unix:///tmp/foo.socket",
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "trace",
env_vars: Vec::new(),
debug_console: false,
dev_mode: false,
log_level: DEFAULT_LOG_LEVEL,
hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: ".trace",
env_vars: Vec::new(),
debug_console: false,
dev_mode: false,
log_level: DEFAULT_LOG_LEVEL,
hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.tracer",
env_vars: Vec::new(),
debug_console: false,
dev_mode: false,
log_level: DEFAULT_LOG_LEVEL,
hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.trac",
env_vars: Vec::new(),
debug_console: false,
dev_mode: false,
log_level: DEFAULT_LOG_LEVEL,
hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "agent.trace",
env_vars: Vec::new(),
debug_console: false,
dev_mode: false,
log_level: DEFAULT_LOG_LEVEL,
hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Isolated,
},
TestData {
contents: "agent.trace=isolated",
env_vars: Vec::new(),
debug_console: false,
dev_mode: false,
log_level: DEFAULT_LOG_LEVEL,
hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Isolated,
},
TestData {
contents: "agent.trace=disabled",
env_vars: Vec::new(),
debug_console: false,
dev_mode: false,
log_level: DEFAULT_LOG_LEVEL,
hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
TestData {
contents: "",
env_vars: vec!["KATA_AGENT_TRACE_TYPE=isolated"],
debug_console: false,
dev_mode: false,
log_level: DEFAULT_LOG_LEVEL,
hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Isolated,
},
TestData {
contents: "",
env_vars: vec!["KATA_AGENT_TRACE_TYPE=disabled"],
debug_console: false,
dev_mode: false,
log_level: DEFAULT_LOG_LEVEL,
hotplug_timeout: DEFAULT_HOTPLUG_TIMEOUT,
container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
server_addr: TEST_SERVER_ADDR,
unified_cgroup_hierarchy: false,
tracing: tracer::TraceType::Disabled,
},
];
@ -967,6 +1169,7 @@ mod tests {
);
assert_eq!(config.container_pipe_size, 0, "{}", msg);
assert_eq!(config.server_addr, TEST_SERVER_ADDR, "{}", msg);
assert_eq!(config.tracing, tracer::TraceType::Disabled, "{}", msg);
let result = config.parse_cmdline(filename);
assert!(result.is_ok(), "{}", msg);
@ -982,6 +1185,7 @@ mod tests {
assert_eq!(d.hotplug_timeout, config.hotplug_timeout, "{}", msg);
assert_eq!(d.container_pipe_size, config.container_pipe_size, "{}", msg);
assert_eq!(d.server_addr, config.server_addr, "{}", msg);
assert_eq!(d.tracing, config.tracing, "{}", msg);
for v in vars_to_unset {
env::remove_var(v);
@ -1378,4 +1582,62 @@ mod tests {
assert_result!(d.result, result, msg);
}
}
#[test]
fn test_get_trace_type() {
#[derive(Debug)]
struct TestData<'a> {
param: &'a str,
result: Result<tracer::TraceType>,
}
let tests = &[
TestData {
param: "",
result: Err(make_err(ERR_INVALID_TRACE_TYPE_PARAM)),
},
TestData {
param: "agent.tracer",
result: Err(make_err(ERR_INVALID_TRACE_TYPE_KEY)),
},
TestData {
param: "agent.trac",
result: Err(make_err(ERR_INVALID_TRACE_TYPE_KEY)),
},
TestData {
param: "agent.trace=",
result: Err(make_err(ERR_INVALID_TRACE_TYPE)),
},
TestData {
param: "agent.trace==",
result: Err(make_err(ERR_INVALID_TRACE_TYPE)),
},
TestData {
param: "agent.trace=foo",
result: Err(make_err(ERR_INVALID_TRACE_TYPE)),
},
TestData {
param: "agent.trace",
result: Ok(tracer::TraceType::Isolated),
},
TestData {
param: "agent.trace=isolated",
result: Ok(tracer::TraceType::Isolated),
},
TestData {
param: "agent.trace=disabled",
result: Ok(tracer::TraceType::Disabled),
},
];
for (i, d) in tests.iter().enumerate() {
let msg = format!("test[{}]: {:?}", i, d);
let result = get_trace_type(d.param);
let msg = format!("{}: result: {:?}", msg, result);
assert_result!(d.result, result, msg);
}
}
}

20
src/agent/src/main.rs
View File

@ -32,7 +32,7 @@ use std::os::unix::io::AsRawFd;
use std::path::Path;
use std::process::exit;
use std::sync::Arc;
use tracing::instrument;
use tracing::{instrument, span};
mod config;
mod console;
@ -56,7 +56,7 @@ mod version;
use mount::{cgroups_mount, general_mount};
use sandbox::Sandbox;
use signal::setup_signal_handler;
use slog::Logger;
use slog::{error, info, o, warn, Logger};
use uevent::watch_uevents;
use futures::future::join_all;
@ -71,6 +71,7 @@ use tokio::{
};
mod rpc;
mod tracer;
const NAME: &str = "kata-agent";
const KERNEL_CMDLINE_FILE: &str = "/proc/cmdline";
@ -201,6 +202,17 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
ttrpc_log_guard = Ok(slog_stdlog::init().map_err(|e| e)?);
}
if config.tracing != tracer::TraceType::Disabled {
let _ = tracer::setup_tracing(NAME, &logger, &config)?;
}
let root = span!(tracing::Level::TRACE, "root-span", work_units = 2);
// XXX: Start the root trace transaction.
//
// XXX: Note that *ALL* spans needs to start after this point!!
let _enter = root.enter();
// Start the sandbox and wait for its ttRPC server to end
start_sandbox(&logger, &config, init_mode, &mut tasks, shutdown_rx.clone()).await?;
@ -229,6 +241,10 @@ async fn real_main() -> std::result::Result<(), Box<dyn std::error::Error>> {
}
}
if config.tracing != tracer::TraceType::Disabled {
tracer::end_tracing();
}
eprintln!("{} shutdown complete", NAME);
Ok(())

91
src/agent/src/tracer.rs Normal file
View File

@ -0,0 +1,91 @@
// Copyright (c) 2020-2021 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
//
use crate::config::AgentConfig;
use anyhow::Result;
use opentelemetry::{global, sdk::trace::Config, trace::TracerProvider};
use slog::{info, o, Logger};
use std::error::Error;
use std::fmt;
use std::str::FromStr;
use tracing_opentelemetry::OpenTelemetryLayer;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::Registry;
#[derive(Debug, PartialEq)]
pub enum TraceType {
Disabled,
Isolated,
}
#[derive(Debug)]
pub struct TraceTypeError {
details: String,
}
impl TraceTypeError {
fn new(msg: &str) -> TraceTypeError {
TraceTypeError {
details: msg.into(),
}
}
}
impl Error for TraceTypeError {}
impl fmt::Display for TraceTypeError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}", self.details)
}
}
impl FromStr for TraceType {
type Err = TraceTypeError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"isolated" => Ok(TraceType::Isolated),
"disabled" => Ok(TraceType::Disabled),
_ => Err(TraceTypeError::new("invalid trace type")),
}
}
}
pub fn setup_tracing(name: &'static str, logger: &Logger, _agent_cfg: &AgentConfig) -> Result<()> {
let logger = logger.new(o!("subsystem" => "vsock-tracer"));
let exporter = vsock_exporter::Exporter::builder()
.with_logger(&logger)
.init();
let config = Config::default();
let builder = opentelemetry::sdk::trace::TracerProvider::builder()
.with_simple_exporter(exporter)
.with_config(config);
let provider = builder.build();
// We don't need a versioned tracer.
let version = None;
let tracer = provider.get_tracer(name, version);
let _global_provider = global::set_tracer_provider(provider);
let layer = OpenTelemetryLayer::new(tracer);
let subscriber = Registry::default().with(layer);
tracing::subscriber::set_global_default(subscriber)?;
info!(logger, "tracing setup");
Ok(())
}
pub fn end_tracing() {
global::shutdown_tracer_provider();
}

19
src/agent/vsock-exporter/Cargo.toml Normal file
View File

@ -0,0 +1,19 @@
[package]
name = "vsock-exporter"
version = "0.1.0"
authors = ["James O. D. Hunt <james.o.hunt@intel.com>"]
edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
nix = "0.20.0"
libc = "0.2.94"
thiserror = "1.0.24"
opentelemetry = { version = "0.14.0", features=["serialize"] }
serde = { version = "1.0.126", features = ["derive"] }
vsock = "0.2.3"
bincode = "1.3.3"
byteorder = "1.4.3"
slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_info"] }
async-trait = "0.1.50"

196
src/agent/vsock-exporter/src/lib.rs Normal file
View File

@ -0,0 +1,196 @@
// Copyright (c) 2020-2021 Intel Corporation
//
// SPDX-License-Identifier: Apache-2.0
//
// The VSOCK Exporter sends trace spans "out" to the forwarder running on the
// host (which then forwards them on to a trace collector). The data is sent
// via a VSOCK socket that the forwarder process is listening on. To allow the
// forwarder to know how much data to each for each trace span the simplest
// protocol is employed which uses a header packet and the payload (trace
// span) data. The header packet is a simple count of the number of bytes in the
// payload, which allows the forwarder to know how many bytes it must read to
// consume the trace span. The payload is a serialised version of the trace span.
use async_trait::async_trait;
use byteorder::{ByteOrder, NetworkEndian};
use opentelemetry::sdk::export::trace::{ExportResult, SpanData, SpanExporter};
use opentelemetry::sdk::export::ExportError;
use slog::{error, o, Logger};
use std::io::{ErrorKind, Write};
use std::net::Shutdown;
use std::sync::Mutex;
use vsock::{SockAddr, VsockStream};
const ANY_CID: &str = "any";
// Must match the value of the variable of the same name in the trace forwarder.
const HEADER_SIZE_BYTES: u64 = std::mem::size_of::<u64>() as u64;
// By default, the VSOCK exporter should talk "out" to the host where the
// forwarder is running.
const DEFAULT_CID: u32 = libc::VMADDR_CID_HOST;
// The VSOCK port the forwarders listens on by default
const DEFAULT_PORT: u32 = 10240;
#[derive(Debug)]
pub struct Exporter {
port: u32,
cid: u32,
conn: Mutex<VsockStream>,
logger: Logger,
}
impl Exporter {
/// Create a new exporter builder.
pub fn builder() -> Builder {
Builder::default()
}
}
#[derive(thiserror::Error, Debug)]
pub enum Error {
#[error("connection error: {0}")]
ConnectionError(String),
#[error("serialisation error: {0}")]
SerialisationError(#[from] bincode::Error),
#[error("I/O error: {0}")]
IOError(#[from] std::io::Error),
}
impl ExportError for Error {
fn exporter_name(&self) -> &'static str {
"vsock-exporter"
}
}
fn make_io_error(desc: String) -> std::io::Error {
std::io::Error::new(ErrorKind::Other, desc)
}
// Send a trace span to the forwarder running on the host.
fn write_span(writer: &mut dyn Write, span: &SpanData) -> Result<(), std::io::Error> {
let encoded_payload: Vec<u8> =
bincode::serialize(&span).map_err(|e| make_io_error(e.to_string()))?;
let payload_len: u64 = encoded_payload.len() as u64;
let mut payload_len_as_bytes: [u8; HEADER_SIZE_BYTES as usize] =
[0; HEADER_SIZE_BYTES as usize];
// Encode the header
NetworkEndian::write_u64(&mut payload_len_as_bytes, payload_len);
// Send the header
writer
.write_all(&payload_len_as_bytes)
.map_err(|e| make_io_error(format!("failed to write trace header: {:?}", e)))?;
writer
.write_all(&encoded_payload)
.map_err(|e| make_io_error(format!("failed to write trace payload: {:?}", e)))
}
fn handle_batch(writer: &mut dyn Write, batch: Vec<SpanData>) -> ExportResult {
for span_data in batch {
write_span(writer, &span_data).map_err(Error::IOError)?;
}
Ok(())
}
#[async_trait]
impl SpanExporter for Exporter {
async fn export(&mut self, batch: Vec<SpanData>) -> ExportResult {
let conn = self.conn.lock();
match conn {
Ok(mut c) => handle_batch(&mut *c, batch),
Err(e) => {
error!(self.logger, "failed to obtain connection";
"error" => format!("{}", e));
return Err(Error::ConnectionError(e.to_string()).into());
}
}
}
fn shutdown(&mut self) {
let conn = match self.conn.lock() {
Ok(conn) => conn,
Err(e) => {
error!(self.logger, "failed to obtain connection";
"error" => format!("{}", e));
return;
}
};
conn.shutdown(Shutdown::Write)
.expect("failed to shutdown VSOCK connection");
}
}
#[derive(Debug)]
pub struct Builder {
port: u32,
cid: u32,
logger: Logger,
}
impl Default for Builder {
fn default() -> Self {
let logger = Logger::root(slog::Discard, o!());
Builder {
cid: DEFAULT_CID,
port: DEFAULT_PORT,
logger,
}
}
}
impl Builder {
pub fn with_cid(self, cid: u32) -> Self {
Builder { cid, ..self }
}
pub fn with_port(self, port: u32) -> Self {
Builder { port, ..self }
}
pub fn with_logger(self, logger: &Logger) -> Self {
Builder {
logger: logger.new(o!()),
..self
}
}
pub fn init(self) -> Exporter {
let Builder { port, cid, logger } = self;
let sock_addr = SockAddr::new_vsock(self.cid, self.port);
let cid_str: String;
if self.cid == libc::VMADDR_CID_ANY {
cid_str = ANY_CID.to_string();
} else {
cid_str = format!("{}", self.cid);
}
let msg = format!(
"failed to connect to VSOCK server (port: {}, cid: {}) - {}",
self.port, cid_str, "ensure trace forwarder is running on host"
);
let conn = VsockStream::connect(&sock_addr).expect(&msg);
Exporter {
port,
cid,
conn: Mutex::new(conn),
logger: logger.new(o!("cid" => cid_str, "port" => port)),
}
}
}

8
src/trace-forwarder/Cargo.lock generated
View File

@ -699,18 +699,18 @@ checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]]
name = "serde"
version = "1.0.114"
version = "1.0.126"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3"
checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.114"
version = "1.0.126"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e"
checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43"
dependencies = [
"proc-macro2",
"quote",

2
src/trace-forwarder/Cargo.toml
View File

@ -15,7 +15,7 @@ clap = "2.33.0"
vsock = "0.2.3"
nix = "0.20.0"
libc = "0.2.94"
serde = { version = "1.0.106", features = ["derive"] }
serde = { version = "1.0.126", features = ["derive"] }
bincode = "1.3.3"
byteorder = "1.4.3"
serde_json = "1.0.44"

3
src/trace-forwarder/src/handler.rs
View File

@ -19,6 +19,9 @@ use vsock::VsockStream;
// This constant defines the number of bytes used to encode the header on the
// wire. In other words, the first 64-bits of the packet contain a number
// specifying how many bytes are in the remainder of the packet.
//
// Must match the value of the variable of the same name in the agents
// vsock-exporter.
const HEADER_SIZE_BYTES: u64 = std::mem::size_of::<u64>() as u64;
fn mk_io_err(msg: &str) -> std::io::Error {