refactor(shim-mgmt): move client side to libs

The client side is moved to libs. This is to solve the problem
that including clients will bring about messy dependencies.

Fixes: #5874
Signed-off-by: Ji-Xinyou <jerryji0414@outlook.com>
This commit is contained in:
Ji-Xinyou 2022-12-14 17:42:25 +08:00
parent 5ef7ed72ae
commit fbf294da3f
19 changed files with 306 additions and 57 deletions

167
src/libs/Cargo.lock generated
View File

@ -40,6 +40,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "base64"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "bitflags"
version = "1.2.1"
@ -197,6 +203,12 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d"
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "futures"
version = "0.3.21"
@ -338,6 +350,82 @@ dependencies = [
"libc",
]
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "http"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"
dependencies = [
"bytes 1.1.0",
"fnv",
"itoa",
]
[[package]]
name = "http-body"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
dependencies = [
"bytes 1.1.0",
"http",
"pin-project-lite",
]
[[package]]
name = "httparse"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
[[package]]
name = "httpdate"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
[[package]]
name = "hyper"
version = "0.14.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c"
dependencies = [
"bytes 1.1.0",
"futures-channel",
"futures-core",
"futures-util",
"http",
"http-body",
"httparse",
"httpdate",
"itoa",
"pin-project-lite",
"socket2",
"tokio",
"tower-service",
"tracing",
"want",
]
[[package]]
name = "hyperlocal"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c"
dependencies = [
"futures-util",
"hex",
"hyper",
"pin-project",
"tokio",
]
[[package]]
name = "indexmap"
version = "1.8.1"
@ -420,6 +508,8 @@ dependencies = [
name = "kata-types"
version = "0.1.0"
dependencies = [
"anyhow",
"base64",
"bitmask-enum",
"byte-unit",
"glob",
@ -639,6 +729,26 @@ dependencies = [
"indexmap",
]
[[package]]
name = "pin-project"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "pin-project-lite"
version = "0.2.8"
@ -947,6 +1057,16 @@ dependencies = [
"syn",
]
[[package]]
name = "shim-interface"
version = "0.1.0"
dependencies = [
"anyhow",
"hyper",
"hyperlocal",
"tokio",
]
[[package]]
name = "slab"
version = "0.4.6"
@ -1002,9 +1122,9 @@ checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
[[package]]
name = "socket2"
version = "0.4.4"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0"
checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd"
dependencies = [
"libc",
"winapi",
@ -1107,6 +1227,7 @@ dependencies = [
"libc",
"memchr",
"mio",
"num_cpus",
"pin-project-lite",
"socket2",
"tokio-macros",
@ -1146,6 +1267,38 @@ dependencies = [
"serde",
]
[[package]]
name = "tower-service"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
[[package]]
name = "tracing"
version = "0.1.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09"
dependencies = [
"cfg-if",
"pin-project-lite",
"tracing-core",
]
[[package]]
name = "tracing-core"
version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f"
dependencies = [
"lazy_static",
]
[[package]]
name = "try-lock"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
[[package]]
name = "ttrpc"
version = "0.6.1"
@ -1214,6 +1367,16 @@ dependencies = [
"nix 0.23.1",
]
[[package]]
name = "want"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0"
dependencies = [
"log",
"try-lock",
]
[[package]]
name = "wasi"
version = "0.9.0+wasi-snapshot-preview1"

View File

@ -6,6 +6,7 @@ members = [
"oci",
"protocols",
"safe-path",
"shim-interface",
"test-utils",
]
resolver = "2"

View File

@ -0,0 +1,18 @@
[package]
name = "shim-interface"
version = "0.1.0"
description = "A library to provide service interface of Kata Containers"
keywords = ["kata", "container", "http"]
categories = ["services"]
authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
repository = "https://github.com/kata-containers/kata-containers.git"
homepage = "https://katacontainers.io/"
readme = "README.md"
license = "Apache-2.0"
edition = "2018"
[dependencies]
anyhow = "^1.0"
tokio = { version = "1.8.0", features = ["rt-multi-thread"] }
hyper = { version = "0.14.20", features = ["stream", "server", "http1"] }
hyperlocal = "0.8"

View File

@ -0,0 +1,66 @@
// Copyright (c) 2022 Alibaba Cloud
//
// SPDX-License-Identifier: Apache-2.0
//
//! shim-interface is a common library for different components of Kata Containers
//! to make function call through services inside the runtime(runtime-rs runtime).
//!
//! Shim management:
//! Currently, inside the shim, there is a shim management server running as the shim
//! starts, working as a RESTful server. To make function call in runtime from another
//! binary, using the utilities provided in this library is one of the methods.
//!
//! You may construct clients by construct a MgmtClient and let is make specific
//! HTTP request to the server. The server inside shim will multiplex the request
//! to its corresponding handler and run certain methods.
use std::path::Path;
use anyhow::{anyhow, Result};
pub mod shim_mgmt;
pub const KATA_PATH: &str = "/run/kata";
pub const SHIM_MGMT_SOCK_NAME: &str = "shim-monitor.sock";
// return sandbox's storage path
pub fn sb_storage_path() -> String {
String::from(KATA_PATH)
}
// returns the address of the unix domain socket(UDS) for communication with shim
// management service using http
// normally returns "unix:///run/kata/{sid}/shim_monitor.sock"
pub fn mgmt_socket_addr(sid: &str) -> Result<String> {
if sid.is_empty() {
return Err(anyhow!(
"Empty sandbox id for acquiring socket address for shim_mgmt"
));
}
let p = Path::new(&sb_storage_path())
.join(sid)
.join(SHIM_MGMT_SOCK_NAME);
if let Some(p) = p.to_str() {
Ok(format!("unix://{}", p))
} else {
Err(anyhow!("Bad socket path"))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_mgmt_socket_addr() {
let sid = "414123";
let addr = mgmt_socket_addr(sid).unwrap();
assert_eq!(addr, "unix:///run/kata/414123/shim-monitor.sock");
let sid = "";
assert!(mgmt_socket_addr(sid).is_err());
}
}

View File

@ -1,3 +1,4 @@
#![allow(dead_code)]
// Copyright (c) 2019-2022 Alibaba Cloud
// Copyright (c) 2019-2022 Ant Group
//
@ -10,7 +11,7 @@
use std::{path::Path, path::PathBuf, time::Duration};
use super::server::mgmt_socket_addr;
use crate::mgmt_socket_addr;
use anyhow::{anyhow, Context, Result};
use hyper::{Body, Client, Method, Request, Response};
use hyperlocal::{UnixClientExt, UnixConnector, Uri};
@ -30,8 +31,8 @@ pub struct MgmtClient {
impl MgmtClient {
/// Construct a new client connecting to shim mgmt server
pub fn new(sid: String, timeout: Option<Duration>) -> Result<Self> {
let unix_socket_path = mgmt_socket_addr(sid);
pub fn new(sid: &str, timeout: Option<Duration>) -> Result<Self> {
let unix_socket_path = mgmt_socket_addr(sid).context("Failed to get unix socket path")?;
let s_addr = unix_socket_path
.strip_prefix("unix:")
.context("failed to strix prefix")?;

View File

@ -0,0 +1,24 @@
// Copyright (c) 2022 Alibaba Cloud
//
// SPDX-License-Identifier: Apache-2.0
//
/// The shim management client module
pub mod client;
/// The key for direct volume path
pub const DIRECT_VOLUME_PATH_KEY: &str = "path";
/// URL for stats direct volume
pub const DIRECT_VOLUME_STATS_URL: &str = "/direct-volume/stats";
/// URL for resizing direct volume
pub const DIRECT_VOLUME_RESIZE_URL: &str = "/direct-volume/resize";
/// URL for querying agent's socket
pub const AGENT_URL: &str = "/agent-url";
/// URL for operation on guest iptable (ipv4)
pub const IP_TABLE_URL: &str = "/iptables";
/// URL for operation on guest iptable (ipv6)
pub const IP6_TABLE_URL: &str = "/ip6tables";
/// URL for querying metrics inside shim
pub const METRICS_URL: &str = "/metrics";
pub const ERR_NO_SHIM_SERVER: &str = "Failed to create shim management server";

View File

@ -28,6 +28,7 @@ rand = "0.8.4"
kata-sys-util = { path = "../../../libs/kata-sys-util" }
kata-types = { path = "../../../libs/kata-types" }
logging = { path = "../../../libs/logging" }
shim-interface = { path = "../../../libs/shim-interface" }
dragonball = { path = "../../../dragonball", features = ["atomic-guest-memory", "virtio-vsock", "hotplug", "virtio-blk", "virtio-net", "virtio-fs"] }

View File

@ -20,7 +20,8 @@ use kata_types::{
capabilities::{Capabilities, CapabilityBits},
config::hypervisor::Hypervisor as HypervisorConfig,
};
use persist::{sandbox_persist::Persist, KATA_PATH};
use persist::sandbox_persist::Persist;
use shim_interface::KATA_PATH;
use std::{collections::HashSet, fs::create_dir_all, path::PathBuf};
const DRAGONBALL_KERNEL: &str = "vmlinux";

View File

@ -14,7 +14,7 @@ use kata_types::capabilities::Capabilities;
use super::inner::DragonballInner;
use crate::{utils, VcpuThreadIds, VmmState};
use persist::KATA_PATH;
use shim_interface::KATA_PATH;
const DEFAULT_HYBRID_VSOCK_NAME: &str = "kata.hvsock";
fn get_vsock_path(root: &str) -> String {

View File

@ -10,6 +10,7 @@ async-trait = "0.1.48"
anyhow = "^1.0"
kata-sys-util = { path = "../../../libs/kata-sys-util"}
kata-types = { path = "../../../libs/kata-types" }
shim-interface = { path = "../../../libs/shim-interface" }
libc = "0.2"
serde = { version = "1.0.138", features = ["derive"] }
serde_json = "1.0.82"

View File

@ -7,9 +7,9 @@
pub mod sandbox_persist;
use anyhow::{anyhow, Context, Ok, Result};
use serde::de;
use shim_interface::KATA_PATH;
use std::{fs::File, io::BufReader};
pub const KATA_PATH: &str = "/run/kata";
pub const PERSIST_FILE: &str = "state.json";
use kata_sys_util::validate::verify_id;
use safe_path::scoped_join;

View File

@ -18,6 +18,7 @@ common = { path = "./common" }
kata-types = { path = "../../../libs/kata-types" }
logging = { path = "../../../libs/logging"}
oci = { path = "../../../libs/oci" }
shim-interface = { path = "../../../libs/shim-interface" }
persist = { path = "../persist" }
hypervisor = { path = "../hypervisor" }
# runtime handler

View File

@ -11,6 +11,6 @@ logging::logger_with_subsystem!(sl, "runtimes");
pub mod manager;
pub use manager::RuntimeHandlerManager;
pub use shim_interface;
mod shim_mgmt;
pub use shim_mgmt::{client::MgmtClient, server::sb_storage_path};
mod static_resource;

View File

@ -19,6 +19,7 @@ use kata_types::{annotations::Annotation, config::TomlConfig};
#[cfg(feature = "linux")]
use linux_container::LinuxContainer;
use persist::sandbox_persist::Persist;
use shim_interface::shim_mgmt::ERR_NO_SHIM_SERVER;
use tokio::sync::{mpsc::Sender, RwLock};
#[cfg(feature = "virt")]
use virt_container::{
@ -117,7 +118,9 @@ impl RuntimeHandlerManagerInner {
let shim_mgmt_svr = MgmtServer::new(
&self.id,
self.runtime_instance.as_ref().unwrap().sandbox.clone(),
);
)
.context(ERR_NO_SHIM_SERVER)?;
tokio::task::spawn(Arc::new(shim_mgmt_svr).run());
info!(sl!(), "shim management http server starts");

View File

@ -12,7 +12,7 @@ use common::Sandbox;
use hyper::{Body, Method, Request, Response, StatusCode};
use std::sync::Arc;
use super::server::{AGENT_URL, IP6_TABLE_URL, IP_TABLE_URL};
use shim_interface::shim_mgmt::{AGENT_URL, IP6_TABLE_URL, IP_TABLE_URL};
// main router for response, this works as a multiplexer on
// http arrival which invokes the corresponding handler function

View File

@ -4,6 +4,11 @@
// SPDX-License-Identifier: Apache-2.0
//
pub mod client;
//! The server side of shim management implementation, receive HTTP
//! requests and multiplex them to corresponding functions inside shim
//!
//! To call services in a RESTful convention, use the client
//! from libs/shim-interface library
mod handlers;
pub mod server;

View File

@ -1,5 +1,5 @@
// Copyright (c) 2019-2022 Alibaba Cloud
// Copyright (c) 2019-2022 Ant Group
// Copyright (c) 2022 Alibaba Cloud
// Copyright (c) 2022 Ant Group
//
// SPDX-License-Identifier: Apache-2.0
//
@ -16,21 +16,11 @@ use std::{fs, path::Path, sync::Arc};
use anyhow::{Context, Result};
use common::Sandbox;
use hyper::{server::conn::Http, service::service_fn};
use persist::KATA_PATH;
use shim_interface::{mgmt_socket_addr, shim_mgmt::ERR_NO_SHIM_SERVER};
use tokio::net::UnixListener;
use super::handlers::handler_mux;
pub(crate) const DIRECT_VOLUMN_PATH_KEY: &str = "path";
pub(crate) const DIRECT_VOLUMN_STATS_URL: &str = "/direct-volumn/stats";
pub(crate) const DIRECT_VOLUMN_RESIZE_URL: &str = "/direct-volumn/resize";
pub(crate) const AGENT_URL: &str = "/agent-url";
pub(crate) const IP_TABLE_URL: &str = "/iptables";
pub(crate) const IP6_TABLE_URL: &str = "/ip6tables";
pub(crate) const METRICS_URL: &str = "/metrics";
const SHIM_MGMT_SOCK_NAME: &str = "shim-monitor.sock";
/// The shim management server instance
pub struct MgmtServer {
/// socket address(with prefix like hvsock://)
@ -42,11 +32,11 @@ pub struct MgmtServer {
impl MgmtServer {
/// construct a new management server
pub fn new(sid: &str, sandbox: Arc<dyn Sandbox>) -> Self {
Self {
s_addr: mgmt_socket_addr(sid.to_owned()),
pub fn new(sid: &str, sandbox: Arc<dyn Sandbox>) -> Result<Self> {
Ok(Self {
s_addr: mgmt_socket_addr(sid).context(ERR_NO_SHIM_SERVER)?,
sandbox,
}
})
}
// TODO(when metrics is supported): write metric addresses to fs
@ -75,21 +65,6 @@ impl MgmtServer {
}
}
// return sandbox's storage path
pub fn sb_storage_path() -> String {
String::from(KATA_PATH)
}
// returns the address of the unix domain socket(UDS) for communication with shim
// management service using http
// normally returns "unix:///run/kata/{sid}/shim_monitor.sock"
pub fn mgmt_socket_addr(sid: String) -> String {
let p = Path::new(&sb_storage_path())
.join(sid)
.join(SHIM_MGMT_SOCK_NAME);
format!("unix://{}", p.to_string_lossy())
}
// from path, return a unix listener corresponding to that path,
// if the path(socket file) is not created, we create that here
async fn listener_from_path(path: String) -> Result<UnixListener> {
@ -104,15 +79,3 @@ async fn listener_from_path(path: String) -> Result<UnixListener> {
info!(sl!(), "mgmt-svr: binding to path {}", path);
UnixListener::bind(file_path).context("bind address")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn mgmt_svr_test_sock_addr() {
let sid = String::from("414123");
let addr = mgmt_socket_addr(sid);
assert_eq!(addr, "unix:///run/kata/414123/shim-monitor.sock");
}
}

View File

@ -16,5 +16,6 @@ ttrpc = { version = "0.6.1" }
common = { path = "../runtimes/common" }
containerd-shim-protos = { version = "0.2.0", features = ["async"]}
logging = { path = "../../../libs/logging"}
shim-interface = { path = "../../../libs/shim-interface" }
runtimes = { path = "../runtimes" }
persist = { path = "../persist" }

View File

@ -28,7 +28,7 @@ use ttrpc::asynchronous::Server;
use crate::task_service::TaskService;
/// message buffer size
const MESSAGE_BUFFER_SIZE: usize = 8;
use persist::KATA_PATH;
use shim_interface::KATA_PATH;
pub struct ServiceManager {
receiver: Option<Receiver<Message>>,