github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-04-27 11:31:05 +00:00
Code Issues Projects Releases Wiki Activity

agent-ctl: Add support to test kata-agent's container creation APIs.

Browse Source 
This commit introduces changes to enable testing kata-agent's container
APIs of CreateContainer/StartContainer/RemoveContainer. The changeset
include:
- using confidential-containers image-rs crate to pull/unpack/mount a
container image. Currently supports only un-authenicated registry pull
- re-factor api handlers to reduce cmdline complexity and handle
request generation logic in tool
- introduce an OCI config template for container creation
- add test case

Fixes #9707

Signed-off-by: Sumedh Alok Sharma <sumsharma@microsoft.com>
This commit is contained in:
Sumedh Alok Sharma 2024-10-09 00:06:57 +05:30
parent 2efcb442f4
commit 4b7aba5c57
11 changed files with 2844 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2603
src/tools/agent-ctl/Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

8
src/tools/agent-ctl/Cargo.toml
View File

@ -28,6 +28,7 @@ slog = "2.7.0"
slog-scope = "4.4.0"
rand = "0.8.4"
protobuf = "3.2.0"
log = "0.4.22"
nix = "0.23.0"
libc = "0.2.112"
@ -41,4 +42,11 @@ humantime = "2.1.0"
serde = { version = "1.0.131", features = ["derive"] }
serde_json = "1.0.73"
# Image pull/unpack
image-rs = { git = "https://github.com/confidential-containers/guest-components", rev = "v0.10.0", features = ["snapshot-overlayfs", "oci-client-rustls", "signature-cosign-rustls"] }
safe-path = { path = "../../libs/safe-path" }
tokio = { version = "1.28.1", features = ["signal"] }
[workspace]

71
src/tools/agent-ctl/src/client.rs
View File

@ -5,7 +5,7 @@
// Description: Client side of ttRPC comms
use crate::types::{Config, CopyFileInput, Options, SetPolicyInput};
use crate::types::*;
use crate::utils;
use anyhow::{anyhow, Result};
use byteorder::ByteOrder;
@ -34,7 +34,7 @@ macro_rules! run_if_auto_values {
let cfg = $ctx.metadata.get(METADATA_CFG_NS);
if let Some(v) = cfg {
if v.contains(&NO_AUTO_VALUES_CFG_NAME.to_string()) {
if v.contains(&AUTO_VALUES_CFG_NAME.to_string()) {
debug!(sl!(), "Running closure to generate values");
$closure()?;
@ -103,9 +103,9 @@ const ERR_API_FAILED: &str = "API failed";
// Value used as a "namespace" in the ttRPC Context's metadata.
const METADATA_CFG_NS: &str = "agent-ctl-cfg";
// Special value which if found means do not generate any values
// Special value which if found means generate any values
// automatically.
const NO_AUTO_VALUES_CFG_NAME: &str = "no-auto-values";
const AUTO_VALUES_CFG_NAME: &str = "auto-values";
static AGENT_CMDS: &[AgentCmd] = &[
AgentCmd {
@ -640,7 +640,7 @@ pub fn client(cfg: &Config, commands: Vec<&str>) -> Result<()> {
// of this option.
if !cfg.no_auto_values {
ttrpc_ctx.add(METADATA_CFG_NS.into(), NO_AUTO_VALUES_CFG_NAME.to_string());
ttrpc_ctx.add(METADATA_CFG_NS.into(), AUTO_VALUES_CFG_NAME.to_string());
debug!(sl!(), "Automatic value generation disabled");
}
@ -921,20 +921,18 @@ fn agent_cmd_sandbox_create(
ctx: &Context,
client: &AgentServiceClient,
_health: &HealthClient,
options: &mut Options,
_options: &mut Options,
args: &str,
) -> Result<()> {
let mut req: CreateSandboxRequest = utils::make_request(args)?;
// Generate sandbox_id if it is empty
if req.sandbox_id.is_empty() {
req.set_sandbox_id(utils::random_sandbox_id());
}
let ctx = clone_context(ctx);
run_if_auto_values!(ctx, || -> Result<()> {
let sid = utils::get_option("sid", options, args)?;
req.set_sandbox_id(sid);
Ok(())
});
debug!(sl!(), "sending request"; "request" => format!("{:?}", req));
let reply = client
@ -974,26 +972,19 @@ fn agent_cmd_container_create(
ctx: &Context,
client: &AgentServiceClient,
_health: &HealthClient,
options: &mut Options,
_options: &mut Options,
args: &str,
) -> Result<()> {
let mut req: CreateContainerRequest = utils::make_request(args)?;
let input: CreateContainerInput = utils::make_request(args)?;
if input.image.is_empty() {
info!(sl!(), "create container: error image is empty");
return Err(anyhow!("CreateContainer needs image reference"));
}
let ctx = clone_context(ctx);
// FIXME: container create: add back "spec=file:///" support
run_if_auto_values!(ctx, || -> Result<()> {
let cid = utils::get_option("cid", options, args)?;
let exec_id = utils::get_option("exec_id", options, args)?;
let ttrpc_spec = utils::get_ttrpc_spec(options, &cid).map_err(|e| anyhow!(e))?;
req.set_container_id(cid);
req.set_exec_id(exec_id);
req.set_OCI(ttrpc_spec);
Ok(())
});
let req = utils::make_create_container_request(input)?;
debug!(sl!(), "sending request"; "request" => format!("{:?}", req));
@ -1011,19 +1002,13 @@ fn agent_cmd_container_remove(
ctx: &Context,
client: &AgentServiceClient,
_health: &HealthClient,
options: &mut Options,
_options: &mut Options,
args: &str,
) -> Result<()> {
let mut req: RemoveContainerRequest = utils::make_request(args)?;
let req: RemoveContainerRequest = utils::make_request(args)?;
let ctx = clone_context(ctx);
run_if_auto_values!(ctx, || -> Result<()> {
let cid = utils::get_option("cid", options, args)?;
req.set_container_id(cid);
Ok(())
});
debug!(sl!(), "sending request"; "request" => format!("{:?}", req));
let reply = client
@ -1033,6 +1018,9 @@ fn agent_cmd_container_remove(
info!(sl!(), "response received";
"response" => format!("{:?}", reply));
// Un-mount the rootfs mount point.
utils::remove_container_image_mount(req.container_id())?;
Ok(())
}
@ -1180,20 +1168,13 @@ fn agent_cmd_container_start(
ctx: &Context,
client: &AgentServiceClient,
_health: &HealthClient,
options: &mut Options,
_options: &mut Options,
args: &str,
) -> Result<()> {
let mut req: StartContainerRequest = utils::make_request(args)?;
let req: StartContainerRequest = utils::make_request(args)?;
let ctx = clone_context(ctx);
run_if_auto_values!(ctx, || -> Result<()> {
let cid = utils::get_option("cid", options, args)?;
req.set_container_id(cid);
Ok(())
});
debug!(sl!(), "sending request"; "request" => format!("{:?}", req));
let reply = client

63
src/tools/agent-ctl/src/image.rs Normal file
View File

@ -0,0 +1,63 @@
// Copyright (c) 2024 Microsoft Corporation
//
// SPDX-License-Identifier: Apache-2.0
//
// Description: Image client to manage container images for testing container creation
use anyhow::{anyhow, Context, Result};
use image_rs::image::ImageClient;
use nix::mount::umount;
use safe_path::scoped_join;
use slog::{debug, warn};
use std::fs;
use std::path::PathBuf;
const IMAGE_WORK_DIR: &str = "/run/kata-containers/test_image/";
const CONTAINER_BASE_TEST: &str = "/run/kata-containers/testing/";
// Pulls the container image referenced in `image` using image-rs
// and returns the bundle path containing the rootfs (mounted by
// the underlying snapshotter, overlayfs in this case) & config.json
// Uses anonymous image registry authentication.
pub fn pull_image(image: &str, cid: &str) -> Result<String> {
if image.is_empty() || cid.is_empty() {
warn!(sl!(), "pull_image: invalid inputs");
return Err(anyhow!(
"Invalid image reference or container id to pull image"
));
}
debug!(sl!(), "pull_image: creating image client");
let mut image_client = ImageClient::new(PathBuf::from(IMAGE_WORK_DIR));
image_client.config.auth = false;
image_client.config.security_validate = false;
// setup the container test base path
fs::create_dir_all(CONTAINER_BASE_TEST)?;
// setup the container bundle path
let bundle_dir = scoped_join(CONTAINER_BASE_TEST, cid)?;
fs::create_dir_all(bundle_dir.clone())?;
// pull the image
let image_id = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?
.block_on(image_client.pull_image(image, &bundle_dir, &None, &None))
.context("pull and unpack container image")?;
debug!(
sl!(),
"pull_image: image pull for {:?} successfull", image_id
);
// return the bundle path created by unpacking the images
Ok(bundle_dir.as_path().display().to_string())
}
pub fn remove_image_mount(cid: &str) -> Result<()> {
let bundle_path = scoped_join(CONTAINER_BASE_TEST, cid)?;
let rootfs_path = scoped_join(bundle_path, "rootfs")?;
umount(&rootfs_path)?;
Ok(())
}

1
src/tools/agent-ctl/src/main.rs
View File

@ -21,6 +21,7 @@ macro_rules! sl {
}
mod client;
mod image;
mod rpc;
mod types;
mod utils;

7
src/tools/agent-ctl/src/types.rs
View File

@ -33,3 +33,10 @@ pub struct CopyFileInput {
pub struct SetPolicyInput {
pub policy_file: String,
}
// CreateContainer input
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct CreateContainerInput {
pub image: String,
pub id: String,
}

80
src/tools/agent-ctl/src/utils.rs
View File

@ -3,20 +3,24 @@
// SPDX-License-Identifier: Apache-2.0
//
use crate::types::{Config, CopyFileInput, Options, SetPolicyInput};
use crate::image;
use crate::types::*;
use anyhow::{anyhow, Result};
use oci::{Root as ociRoot, Spec as ociSpec};
use oci_spec::runtime as oci;
use protocols::agent::{CopyFileRequest, SetPolicyRequest};
use protocols::oci::{Mount as ttrpcMount, Root as ttrpcRoot, Spec as ttrpcSpec};
use protocols::agent::{CopyFileRequest, CreateContainerRequest, SetPolicyRequest};
use protocols::oci::{
Mount as ttrpcMount, Process as ttrpcProcess, Root as ttrpcRoot, Spec as ttrpcSpec,
};
use rand::Rng;
use safe_path::scoped_join;
use serde::de::DeserializeOwned;
use slog::{debug, warn};
use std::collections::HashMap;
use std::fs::{self, File};
use std::io::Read;
use std::os::unix::fs::MetadataExt;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
// Length of a sandbox identifier
@ -30,6 +34,10 @@ const MIN_HOSTNAME_LEN: u8 = 8;
// Name of the OCI configuration file found at the root of an OCI bundle.
const CONFIG_FILE: &str = "config.json";
// Path to OCI configuration template
const OCI_CONFIG_TEMPLATE: &str =
"/opt/kata/share/defaults/kata-containers/agent-ctl/oci_config.json";
lazy_static! {
// Create a mutable hash map statically
static ref SIGNALS: Arc<Mutex<HashMap<&'static str, u8>>> = {
@ -492,3 +500,67 @@ pub fn make_set_policy_request(input: &SetPolicyInput) -> Result<SetPolicyReques
req.set_policy(policy_data);
Ok(req)
}
fn fix_oci_process_args(spec: &mut ttrpcSpec, bundle: &str) -> Result<()> {
let config_path = scoped_join(bundle, CONFIG_FILE)?;
let file = File::open(config_path)?;
let oci_from_config: ociSpec = serde_json::from_reader(file)?;
let mut process: ttrpcProcess = match &oci_from_config.process() {
Some(p) => p.clone().into(),
None => {
return Err(anyhow!("Failed to set container process args"));
}
};
spec.take_Process().set_Args(process.take_Args());
Ok(())
}
// Helper function to generate create container request
pub fn make_create_container_request(
input: CreateContainerInput,
) -> Result<CreateContainerRequest> {
// read in the oci configuration template
if !Path::new(OCI_CONFIG_TEMPLATE).exists() {
warn!(sl!(), "make_create_container_request: Missig template file");
return Err(anyhow!("Missing OCI Config template file"));
}
let file = File::open(OCI_CONFIG_TEMPLATE)?;
let spec: ociSpec = serde_json::from_reader(file)?;
let mut req = CreateContainerRequest::default();
let c_id = if !input.id.is_empty() {
input.id
} else {
random_container_id()
};
debug!(
sl!(),
"make_create_container_request: pulling container image"
);
// Pull and unpack the container image
let bundle = image::pull_image(&input.image, &c_id)?;
let mut ttrpc_spec = oci_to_ttrpc(&bundle, &c_id, &spec)?;
// Rootfs has been handled with bundle after pulling image
// Fix the container process argument.
fix_oci_process_args(&mut ttrpc_spec, &bundle)?;
req.set_container_id(c_id);
req.set_OCI(ttrpc_spec);
debug!(sl!(), "CreateContainer request generated successfully");
Ok(req)
}
pub fn remove_container_image_mount(c_id: &str) -> Result<()> {
image::remove_image_mount(c_id)
}

170
src/tools/agent-ctl/template/oci_config.json Normal file
View File

@ -0,0 +1,170 @@
{
"ociVersion": "1.1.0-rc.1-test",
"process": {
"user": {
},
"args": [
""
],
"env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"TERM=xterm"
],
"cwd": "/",
"capabilities": {
"bounding": [
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FSETID",
"CAP_FOWNER",
"CAP_MKNOD",
"CAP_NET_RAW",
"CAP_SETGID",
"CAP_SETUID",
"CAP_SETFCAP",
"CAP_SETPCAP",
"CAP_NET_BIND_SERVICE",
"CAP_SYS_CHROOT",
"CAP_KILL",
"CAP_AUDIT_WRITE"
],
"effective": [
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FSETID",
"CAP_FOWNER",
"CAP_MKNOD",
"CAP_NET_RAW",
"CAP_SETGID",
"CAP_SETUID",
"CAP_SETFCAP",
"CAP_SETPCAP",
"CAP_NET_BIND_SERVICE",
"CAP_SYS_CHROOT",
"CAP_KILL",
"CAP_AUDIT_WRITE"
],
"permitted": [
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FSETID",
"CAP_FOWNER",
"CAP_MKNOD",
"CAP_NET_RAW",
"CAP_SETGID",
"CAP_SETUID",
"CAP_SETFCAP",
"CAP_SETPCAP",
"CAP_NET_BIND_SERVICE",
"CAP_SYS_CHROOT",
"CAP_KILL",
"CAP_AUDIT_WRITE"
]
},
"noNewPrivileges": true
},
"root": {
"path": "rootfs",
"readonly": true
},
"mounts": [
{
"destination": "/proc",
"type": "proc",
"source": "proc",
"options": [
"nosuid",
"noexec",
"nodev"
]
},
{
"destination": "/dev",
"type": "tmpfs",
"source": "tmpfs",
"options": [
"nosuid",
"strictatime",
"mode=755",
"size=65536k"
]
},
{
"destination": "/dev/pts",
"type": "devpts",
"source": "devpts",
"options": [
"nosuid",
"noexec",
"newinstance",
"ptmxmode=0666",
"mode=0620",
"gid=5"
]
},
{
"destination": "/dev/shm",
"type": "tmpfs",
"source": "shm",
"options": [
"nosuid",
"noexec",
"nodev",
"mode=1777",
"size=65536k"
]
},
{
"destination": "/dev/mqueue",
"type": "mqueue",
"source": "mqueue",
"options": [
"nosuid",
"noexec",
"nodev"
]
},
{
"destination": "/sys",
"type": "sysfs",
"source": "sysfs",
"options": [
"nosuid",
"noexec",
"nodev",
"ro"
]
}
],
"annotations": {
"io.katacontainers.pkg.oci.container_type": "pod_container",
"io.kubernetes.cri.sandbox-id": "",
"io.kubernetes.cri.image-name": "" ,
"io.kubernetes.cri.container-type": "container",
"io.kubernetes.cri.container-name": "",
"io.kubernetes.cri.sandbox-namespace": "default",
"io.kubernetes.cri.sandbox-name": ""
},
"linux": {
"resources": {
},
"namespaces": [
{
"type": "pid"
},
{
"type": "network"
},
{
"type": "ipc"
},
{
"type": "uts"
},
{
"type": "mount"
}
]
}
}

43
tests/functional/kata-agent-apis/api-tests/test_create_container.bats Executable file
View File

@ -0,0 +1,43 @@
#!/usr/bin/env bats
# Copyright (c) 2024 Microsoft Corporation
#
# SPDX-License-Identifier: Apache-2.0
load "${BATS_TEST_DIRNAME}/../../../common.bash"
load "${BATS_TEST_DIRNAME}/../setup_common.sh"
setup_file() {
info "setup"
}
@test "Test CreateContainer API: Create a container" {
info "Create a container"
sandbox_id=$RANDOM
container_id="test_container_${RANDOM}"
local cmds=()
cmds+="-c 'CreateSandbox json://{\"sandbox_id\": \"$sandbox_id\"}'"
run_agent_ctl "${cmds[@]}"
local image="ghcr.io/linuxcontainers/alpine:latest"
local cmds=()
cmds+="-c 'CreateContainer json://{\"image\": \"$image\", \"id\": \"$container_id\"}'"
run_agent_ctl "${cmds[@]}"
info "Container created successfully."
local cmds=()
cmds+="-c 'StartContainer json://{\"container_id\": \"$container_id\"}'"
run_agent_ctl "${cmds[@]}"
info "Container process started"
local cmds=()
cmds+="-c 'RemoveContainer json://{\"container_id\": \"$container_id\"}'"
run_agent_ctl "${cmds[@]}"
info "Container removed."
}
teardown_file() {
info "teardown"
sudo rm -r /run/kata-containers/ || echo "Failed to clean /run/kata-containers"
}

2
tests/functional/kata-agent-apis/setup_common.sh
View File

@ -79,7 +79,7 @@ run_agent_ctl()
[ -n "$cmds" ] || die "need commands for agent control tool"
local redirect="&>\"${ctl_log_file}\""
local redirect=">> ${ctl_log_file} 2>&1"
local server_address="--server-address ${local_agent_server_addr}"

6
tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh
View File

@ -1008,6 +1008,12 @@ install_tools_helper() {
binary_permissions="$default_binary_permissions"
fi
if [[ "${tool}" == "agent-ctl" ]]; then
defaults_path="${destdir}/opt/kata/share/defaults/kata-containers/agent-ctl"
mkdir -p "${defaults_path}"
install -D --mode 0644 ${repo_root_dir}/src/tools/${tool}/template/oci_config.json "${defaults_path}/oci_config.json"
fi
info "Install static ${tool_binary}"
mkdir -p "${destdir}/opt/kata/bin/"
install -D --mode ${binary_permissions} ${binary} "${destdir}/opt/kata/bin/${tool_binary}"