libs/types: support load Kata runtime configuration from file

Add structures to load Kata runtime configuration from configuration
files. Also define a mechanism for vendor to extend the Kata
configuration structure.

Signed-off-by: Liu Jiang <gerry@linux.alibaba.com>
Signed-off-by: Zhongtao Hu <zhongtaohu.tim@linux.alibaba.com>

src/libs/Cargo.lock

@@ -287,9 +287,13 @@ checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
 name = "kata-types"
 version = "0.1.0"
 dependencies = [
+ "lazy_static",
  "oci",
  "serde",
+ "slog",
+ "slog-scope",
  "thiserror",
+ "toml",
 ]
 
 [[package]]
@@ -802,6 +806,15 @@ dependencies = [
  "vsock",
 ]
 
+[[package]]
+name = "toml"
+version = "0.5.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "ttrpc"
 version = "0.5.2"

src/libs/kata-types/Cargo.toml

@@ -11,9 +11,16 @@ license = "Apache-2.0"
 edition = "2018"
 
 [dependencies]
+lazy_static = "1.4.0"
 serde = { version = "1.0.100", features = ["derive"] }
+slog = "2.5.2"
+slog-scope = "4.4.0"
 thiserror = "1.0"
+toml = "0.5.8"
 
 oci = { path = "../oci" }
 
 [dev-dependencies]
+[features]
+default = []
+enable-vendor = []

src/libs/kata-types/src/config/default.rs

@@ -0,0 +1,23 @@
+// Copyright (c) 2021 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+//! Default configuration values.
+#![allow(missing_docs)]
+
+use lazy_static::lazy_static;
+
+lazy_static! {
+    /// Default configuration file paths.
+    pub static ref DEFAULT_RUNTIME_CONFIGURATIONS: Vec::<&'static str> = vec![
+        "/etc/kata-containers2/configuration.toml",
+        "/usr/share/defaults/kata-containers2/configuration.toml",
+        "/etc/kata-containers/configuration_v2.toml",
+        "/usr/share/defaults/kata-containers/configuration_v2.toml",
+        "/etc/kata-containers/configuration.toml",
+        "/usr/share/defaults/kata-containers/configuration.toml",
+    ];
+}
+
+pub const DEFAULT_INTERNETWORKING_MODEL: &str = "tcfilter";

src/libs/kata-types/src/config/mod.rs

@@ -0,0 +1,125 @@
+// Copyright (c) 2019-2021 Ant Financial
+// Copyright (c) 2019-2021 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::fs;
+use std::io::{self, Result};
+use std::path::{Path, PathBuf};
+
+use crate::sl;
+
+/// Default configuration values.
+pub mod default;
+
+mod runtime;
+pub use self::runtime::{Runtime, RuntimeVendor};
+
+/// Trait to manipulate global Kata configuration information.
+pub trait ConfigOps {
+    /// Adjust the configuration information after loading from configuration file.
+    fn adjust_configuration(_conf: &mut TomlConfig) -> Result<()> {
+        Ok(())
+    }
+
+    /// Validate the configuration information.
+    fn validate(_conf: &TomlConfig) -> Result<()> {
+        Ok(())
+    }
+}
+
+/// Trait to manipulate global Kata configuration information.
+pub trait ConfigObjectOps {
+    /// Adjust the configuration information after loading from configuration file.
+    fn adjust_configuration(&mut self) -> Result<()> {
+        Ok(())
+    }
+
+    /// Validate the configuration information.
+    fn validate(&self) -> Result<()> {
+        Ok(())
+    }
+}
+
+/// Kata configuration information.
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct TomlConfig {
+    /// Kata runtime configuration information.
+    #[serde(default)]
+    pub runtime: Runtime,
+}
+
+impl TomlConfig {
+    /// Load Kata configuration information from configuration files.
+    ///
+    /// If `config_file` is valid, it will used, otherwise a built-in default path list will be
+    /// scanned.
+    pub fn load_from_file<P: AsRef<Path>>(config_file: P) -> Result<(TomlConfig, PathBuf)> {
+        let file_path = if !config_file.as_ref().as_os_str().is_empty() {
+            fs::canonicalize(config_file)?
+        } else {
+            Self::get_default_config_file()?
+        };
+
+        info!(
+            sl!(),
+            "load configuration from: {}",
+            file_path.to_string_lossy()
+        );
+        let content = fs::read_to_string(&file_path)?;
+        let config = Self::load(&content)?;
+
+        Ok((config, file_path))
+    }
+
+    /// Load raw Kata configuration information from configuration files.
+    ///
+    /// If `config_file` is valid, it will used, otherwise a built-in default path list will be
+    /// scanned.
+    pub fn load_raw_from_file<P: AsRef<Path>>(config_file: P) -> Result<(TomlConfig, PathBuf)> {
+        let file_path = if !config_file.as_ref().as_os_str().is_empty() {
+            fs::canonicalize(config_file)?
+        } else {
+            Self::get_default_config_file()?
+        };
+
+        info!(
+            sl!(),
+            "load configuration from: {}",
+            file_path.to_string_lossy()
+        );
+        let content = fs::read_to_string(&file_path)?;
+        let config: TomlConfig = toml::from_str(&content)?;
+
+        Ok((config, file_path))
+    }
+
+    /// Load Kata configuration information from string.
+    pub fn load(content: &str) -> Result<TomlConfig> {
+        let mut config: TomlConfig = toml::from_str(content)?;
+
+        Runtime::adjust_configuration(&mut config)?;
+        info!(sl!(), "get kata config: {:?}", config);
+
+        Ok(config)
+    }
+
+    /// Validate Kata configuration information.
+    pub fn validate(&self) -> Result<()> {
+        Runtime::validate(self)?;
+
+        Ok(())
+    }
+
+    ///  Probe configuration file according to the default configuration file list.
+    fn get_default_config_file() -> Result<PathBuf> {
+        for f in default::DEFAULT_RUNTIME_CONFIGURATIONS.iter() {
+            if let Ok(path) = fs::canonicalize(f) {
+                return Ok(path);
+            }
+        }
+
+        Err(io::Error::from(io::ErrorKind::NotFound))
+    }
+}

src/libs/kata-types/src/config/runtime.rs

@@ -0,0 +1,272 @@
+// Copyright (c) 2019-2021 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::io::Result;
+use std::path::Path;
+
+use super::default;
+use crate::config::{ConfigOps, TomlConfig};
+use crate::{eother, resolve_path, validate_path};
+
+/// Kata runtime configuration information.
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct Runtime {
+    /// If enabled, the runtime will log additional debug messages to the system log.
+    #[serde(default, rename = "enable_debug")]
+    pub debug: bool,
+
+    /// Enabled experimental feature list, format: ["a", "b"].
+    ///
+    /// Experimental features are features not stable enough for production, they may break
+    /// compatibility, and are prepared for a big version bump.
+    #[serde(default)]
+    pub experimental: Vec<String>,
+
+    /// Determines how the VM should be connected to the container network interface.
+    ///
+    /// Options:
+    /// - macvtap: used when the Container network interface can be bridged using macvtap.
+    /// - none: used when customize network. Only creates a tap device. No veth pair.
+    /// - tcfilter: uses tc filter rules to redirect traffic from the network interface provided
+    ///   by plugin to a tap interface connected to the VM.
+    #[serde(default)]
+    pub internetworking_model: String,
+
+    /// If enabled, the runtime won't create a network namespace for shim and hypervisor processes.
+    ///
+    /// This option may have some potential impacts to your host. It should only be used when you
+    /// know what you're doing.
+    ///
+    /// `disable_new_netns` conflicts with `internetworking_model=tcfilter` and
+    /// `internetworking_model=macvtap`. It works only with `internetworking_model=none`.
+    /// The tap device will be in the host network namespace and can connect to a bridge (like OVS)
+    /// directly.
+    ///
+    /// If you are using docker, `disable_new_netns` only works with `docker run --net=none`
+    #[serde(default)]
+    pub disable_new_netns: bool,
+
+    /// If specified, sandbox_bind_mounts identifies host paths to be mounted into the sandboxes
+    /// shared path.
+    ///
+    /// This is only valid if filesystem sharing is utilized. The provided path(s) will be bind
+    /// mounted into the shared fs directory. If defaults are utilized, these mounts should be
+    /// available in the guest at `/run/kata-containers/shared/containers/passthrough/sandbox-mounts`.
+    /// These will not be exposed to the container workloads, and are only provided for potential
+    /// guest services.
+    #[serde(default)]
+    pub sandbox_bind_mounts: Vec<String>,
+
+    /// If enabled, the runtime will add all the kata processes inside one dedicated cgroup.
+    ///
+    /// The container cgroups in the host are not created, just one single cgroup per sandbox.
+    /// The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
+    /// The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
+    /// The sandbox cgroup is constrained if there is no container type annotation.
+    /// See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType
+    #[serde(default)]
+    pub sandbox_cgroup_only: bool,
+
+    /// If enabled, the runtime will create opentracing.io traces and spans.
+    /// See https://www.jaegertracing.io/docs/getting-started.
+    #[serde(default)]
+    pub enable_tracing: bool,
+    /// The full url to the Jaeger HTTP Thrift collector.
+    #[serde(default)]
+    pub jaeger_endpoint: String,
+    /// The username to be used if basic auth is required for Jaeger.
+    #[serde(default)]
+    pub jaeger_user: String,
+    /// The password to be used if basic auth is required for Jaeger.
+    #[serde(default)]
+    pub jaeger_password: String,
+
+    /// If enabled, user can run pprof tools with shim v2 process through kata-monitor.
+    #[serde(default)]
+    pub enable_pprof: bool,
+
+    /// Determines whether container seccomp profiles are passed to the virtual machine and
+    /// applied by the kata agent. If set to true, seccomp is not applied within the guest.
+    #[serde(default)]
+    pub disable_guest_seccomp: bool,
+
+    /// Determines how VFIO devices should be be presented to the container.
+    ///
+    /// Options:
+    /// - vfio: Matches behaviour of OCI runtimes (e.g. runc) as much as possible.  VFIO devices
+    ///   will appear in the container as VFIO character devices under /dev/vfio. The exact names
+    ///   may differ from the host (they need to match the VM's IOMMU group numbers rather than
+    ///   the host's)
+    /// - guest-kernel: This is a Kata-specific behaviour that's useful in certain cases.
+    ///   The VFIO device is managed by whatever driver in the VM kernel claims it. This means
+    ///   it will appear as one or more device nodes or network interfaces depending on the nature
+    ///   of the device. Using this mode requires specially built workloads that know how to locate
+    ///   the relevant device interfaces within the VM.
+    #[serde(default)]
+    pub vfio_mode: String,
+
+    /// Vendor customized runtime configuration.
+    #[serde(default, flatten)]
+    pub vendor: RuntimeVendor,
+}
+
+impl ConfigOps for Runtime {
+    fn adjust_configuration(conf: &mut TomlConfig) -> Result<()> {
+        RuntimeVendor::adjust_configuration(conf)?;
+
+        if conf.runtime.internetworking_model.is_empty() {
+            conf.runtime.internetworking_model = default::DEFAULT_INTERNETWORKING_MODEL.to_owned();
+        }
+
+        for bind in conf.runtime.sandbox_bind_mounts.iter_mut() {
+            resolve_path!(*bind, "sandbox bind mount `{}` is invalid: {}")?;
+        }
+
+        Ok(())
+    }
+
+    fn validate(conf: &TomlConfig) -> Result<()> {
+        RuntimeVendor::validate(conf)?;
+
+        let net_model = &conf.runtime.internetworking_model;
+        if !net_model.is_empty()
+            && net_model != "macvtap"
+            && net_model != "none"
+            && net_model != "tcfilter"
+        {
+            return Err(eother!(
+                "Invalid internetworking_model `{}` in configuration file",
+                net_model
+            ));
+        }
+
+        let vfio_mode = &conf.runtime.vfio_mode;
+        if !vfio_mode.is_empty() && vfio_mode != "vfio" && vfio_mode != "guest-kernel" {
+            return Err(eother!(
+                "Invalid vfio_mode `{}` in configuration file",
+                vfio_mode
+            ));
+        }
+
+        for bind in conf.runtime.sandbox_bind_mounts.iter() {
+            validate_path!(*bind, "sandbox bind mount `{}` is invalid: {}")?;
+        }
+
+        Ok(())
+    }
+}
+
+impl Runtime {
+    /// Check whether experiment `feature` is enabled or not.
+    pub fn is_experiment_enabled(&self, feature: &str) -> bool {
+        self.experimental.contains(&feature.to_string())
+    }
+}
+
+#[cfg(not(feature = "enable-vendor"))]
+mod vendor {
+    use super::*;
+
+    /// Vendor customization runtime configuration.
+    #[derive(Debug, Default, Deserialize, Serialize)]
+    pub struct RuntimeVendor {}
+
+    impl ConfigOps for RuntimeVendor {}
+}
+
+#[cfg(feature = "enable-vendor")]
+#[path = "runtime_vendor.rs"]
+mod vendor;
+
+pub use vendor::RuntimeVendor;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_invalid_config() {
+        let content = r#"
+[runtime]
+enable_debug = 10
+"#;
+        TomlConfig::load(content).unwrap_err();
+
+        let content = r#"
+[runtime]
+enable_debug = true
+internetworking_model = "test"
+"#;
+        let config: TomlConfig = TomlConfig::load(content).unwrap();
+        config.validate().unwrap_err();
+
+        let content = r#"
+[runtime]
+enable_debug = true
+internetworking_model = "macvtap,none"
+"#;
+        let config: TomlConfig = TomlConfig::load(content).unwrap();
+        config.validate().unwrap_err();
+
+        let content = r#"
+[runtime]
+enable_debug = true
+vfio_mode = "none"
+"#;
+        let config: TomlConfig = TomlConfig::load(content).unwrap();
+        config.validate().unwrap_err();
+
+        let content = r#"
+[runtime]
+enable_debug = true
+vfio_mode = "vfio,guest-kernel"
+"#;
+        let config: TomlConfig = TomlConfig::load(content).unwrap();
+        config.validate().unwrap_err();
+
+        let content = r#"
+[runtime]
+enable_debug = true
+vfio_mode = "guest_kernel"
+"#;
+        let config: TomlConfig = TomlConfig::load(content).unwrap();
+        config.validate().unwrap_err();
+    }
+
+    #[test]
+    fn test_config() {
+        let content = r#"
+[runtime]
+enable_debug = true
+experimental = ["a", "b"]
+internetworking_model = "macvtap"
+disable_new_netns = true
+sandbox_bind_mounts = []
+sandbox_cgroup_only = true
+enable_tracing = true
+jaeger_endpoint = "localhost:1234"
+jaeger_user = "user"
+jaeger_password = "pw"
+enable_pprof = true
+disable_guest_seccomp = true
+vfio_mode = "vfio"
+field_should_be_ignored = true
+"#;
+        let config: TomlConfig = TomlConfig::load(content).unwrap();
+        config.validate().unwrap();
+        assert!(config.runtime.debug);
+        assert_eq!(config.runtime.experimental.len(), 2);
+        assert_eq!(&config.runtime.experimental[0], "a");
+        assert_eq!(&config.runtime.experimental[1], "b");
+        assert_eq!(&config.runtime.internetworking_model, "macvtap");
+        assert!(config.runtime.disable_new_netns);
+        assert_eq!(config.runtime.sandbox_bind_mounts.len(), 0);
+        assert!(config.runtime.sandbox_cgroup_only);
+        assert!(config.runtime.enable_tracing);
+        assert!(config.runtime.is_experiment_enabled("a"));
+        assert!(config.runtime.is_experiment_enabled("b"));
+        assert!(!config.runtime.is_experiment_enabled("c"));
+    }
+}

src/libs/kata-types/src/config/runtime_vendor.rs

@@ -0,0 +1,88 @@
+// Copyright (c) 2021 Alibaba Cloud
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+//! A sample for vendor to customize the runtime implementation.
+
+use super::*;
+use crate::{eother, sl};
+use slog::Level;
+/// Vendor customization runtime configuration.
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct RuntimeVendor {
+    /// Log level
+    #[serde(default)]
+    pub log_level: u32,
+
+    /// Prefix for log messages
+    #[serde(default)]
+    pub log_prefix: String,
+}
+
+impl ConfigOps for RuntimeVendor {
+    fn adjust_configuration(conf: &mut TomlConfig) -> Result<()> {
+        if conf.runtime.vendor.log_level > Level::Debug as u32 {
+            conf.runtime.debug = true;
+        }
+
+        Ok(())
+    }
+
+    /// Validate the configuration information.
+    fn validate(conf: &TomlConfig) -> Result<()> {
+        if conf.runtime.vendor.log_level > 10 {
+            warn!(
+                sl!(),
+                "log level {} in configuration file is invalid", conf.runtime.vendor.log_level
+            );
+            return Err(eother!(
+                "log level {} in configuration file is invalid",
+                conf.runtime.vendor.log_level
+            ));
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_invalid_vendor_config() {
+        let content = r#"
+[runtime]
+debug = false
+log_level = 20
+log_prefix = "test"
+"#;
+        let config: TomlConfig = TomlConfig::load(content).unwrap();
+        config.validate().unwrap_err();
+
+        let content = r#"
+[runtime]
+debug = false
+log_level = "test"
+log_prefix = "test"
+"#;
+        TomlConfig::load(content).unwrap_err();
+    }
+
+    #[test]
+    fn test_vendor_config() {
+        let content = r#"
+[runtime]
+debug = false
+log_level = 10
+log_prefix = "test"
+log_fmt = "nouse"
+"#;
+        let config: TomlConfig = TomlConfig::load(content).unwrap();
+        config.validate().unwrap();
+        assert!(config.runtime.debug);
+        assert_eq!(config.runtime.vendor.log_level, 10);
+        assert_eq!(&config.runtime.vendor.log_prefix, "test");
+    }
+}

src/libs/kata-types/src/lib.rs

@@ -5,11 +5,18 @@
 
 //! Constants and Data Types shared by Kata Containers components.
 
-#[deny(missing_docs)]
+#![deny(missing_docs)]
+#[macro_use]
+extern crate slog;
+#[macro_use]
+extern crate serde;
 
 /// Constants and data types annotations.
 pub mod annotations;
 
+/// Kata configuration information from configuration file.
+pub mod config;
+
 /// Constants and data types related to container.
 pub mod container;
 
@@ -18,3 +25,56 @@ pub mod k8s;
 
 /// Constants and data types related to mount point.
 pub mod mount;
+
+/// Convenience macro to obtain the scoped logger
+#[macro_export]
+macro_rules! sl {
+    () => {
+        slog_scope::logger()
+    };
+}
+
+/// Helper to create std::io::Error(std::io::ErrorKind::Other)
+#[macro_export]
+macro_rules! eother {
+    () => (std::io::Error::new(std::io::ErrorKind::Other, ""));
+    ($fmt:expr) => ({
+        std::io::Error::new(std::io::ErrorKind::Other, format!($fmt))
+    });
+    ($fmt:expr, $($arg:tt)*) => ({
+        std::io::Error::new(std::io::ErrorKind::Other, format!($fmt, $($arg)*))
+    });
+}
+
+/// Resolve a path to its final value.
+#[macro_export]
+macro_rules! resolve_path {
+    ($field:expr, $fmt:expr) => {{
+        if !$field.is_empty() {
+            match Path::new(&$field).canonicalize() {
+                Err(e) => Err(eother!($fmt, &$field, e)),
+                Ok(path) => {
+                    $field = path.to_string_lossy().to_string();
+                    Ok(())
+                }
+            }
+        } else {
+            Ok(())
+        }
+    }};
+}
+
+/// Validate a path.
+#[macro_export]
+macro_rules! validate_path {
+    ($field:expr, $fmt:expr) => {{
+        if !$field.is_empty() {
+            Path::new(&$field)
+                .canonicalize()
+                .map_err(|e| eother!($fmt, &$field, e))
+                .map(|_| ())
+        } else {
+            Ok(())
+        }
+    }};
+}