From 85f4e7caf687b8fa40c4c25ef6d00b4ff1d0cd7e Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Tue, 26 Jul 2022 16:08:59 +0800 Subject: [PATCH 01/16] runtime: explicitly mark the source of the log is from qemu.log In qemu.StopVM(), if debug is enabled, the shim will dump logs from qemu.log, but users don't know which logs are from qemu.log and shim itself. Adding some additional messages will help users to distinguish these logs. Fixes: #4745 Signed-off-by: Bin Liu --- src/runtime/virtcontainers/qemu.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 656548e88c..1acc48cac7 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -990,7 +990,7 @@ func (q *qemu) StopVM(ctx context.Context, waitOnly bool) error { if err == nil { scanner := bufio.NewScanner(f) for scanner.Scan() { - q.Logger().Debug(scanner.Text()) + q.Logger().WithField("file", q.qemuConfig.LogFile).Debug(scanner.Text()) } if err := scanner.Err(); err != nil { q.Logger().WithError(err).Debug("read qemu log failed") From 845c1c03cfbe8435ac383964ef243da252726351 Mon Sep 17 00:00:00 2001 From: Xuewei Niu Date: Fri, 29 Jul 2022 18:35:10 +0800 Subject: [PATCH 02/16] agent: use rtnetlink's neighbours API to add neighbors Bump rtnetlink version from 0.8.0 to 0.11.0. Use rtnetlinks's API to add neighbors and fix issues to adapt new verson of rtnetlink. Fixes: #4607 Signed-off-by: Xuewei Niu --- src/agent/Cargo.lock | 81 ++++++++++++++++++++++------------------ src/agent/Cargo.toml | 2 +- src/agent/src/netlink.rs | 60 ++++------------------------- 3 files changed, 53 insertions(+), 90 deletions(-) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 8c9524c9c1..abfca3a780 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -588,8 +588,8 @@ dependencies = [ "libc", "log", "logging", - "netlink-packet-utils", - "netlink-sys", + "netlink-packet-utils 0.4.1", + "netlink-sys 0.7.0", "nix 0.23.1", "oci", "opentelemetry", @@ -734,28 +734,28 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "netlink-packet-core" -version = "0.2.4" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac48279d5062bdf175bdbcb6b58ff1d6b0ecd54b951f7a0ff4bc0550fe903ccb" +checksum = "345b8ab5bd4e71a2986663e88c56856699d060e78e152e6e9d7966fcd5491297" dependencies = [ "anyhow", "byteorder", "libc", - "netlink-packet-utils", + "netlink-packet-utils 0.5.1", ] [[package]] name = "netlink-packet-route" -version = "0.8.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76aed5d3b6e3929713bf1e1334a11fd65180b6d9f5d7c8572664c48b122604f8" +checksum = "f5dee5ed749373c298237fe694eb0a51887f4cc1a27370c8464bac4382348f1a" dependencies = [ "anyhow", "bitflags", "byteorder", "libc", "netlink-packet-core", - "netlink-packet-utils", + "netlink-packet-utils 0.5.1", ] [[package]] @@ -771,18 +771,30 @@ dependencies = [ ] [[package]] -name = "netlink-proto" -version = "0.7.0" +name = "netlink-packet-utils" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd06e90449ae973fe3888c1ff85949604ef5189b4ac9a2ae39518da1e00762d" +checksum = "25af9cf0dc55498b7bd94a1508af7a78706aa0ab715a73c5169273e03c84845e" +dependencies = [ + "anyhow", + "byteorder", + "paste", + "thiserror", +] + +[[package]] +name = "netlink-proto" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65b4b14489ab424703c092062176d52ba55485a89c076b4f9db05092b7223aa6" dependencies = [ "bytes 1.1.0", "futures", "log", "netlink-packet-core", - "netlink-sys", + "netlink-sys 0.8.3", + "thiserror", "tokio", - "tokio-util", ] [[package]] @@ -797,6 +809,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "netlink-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92b654097027250401127914afb37cb1f311df6610a9891ff07a757e94199027" +dependencies = [ + "bytes 1.1.0", + "futures", + "libc", + "log", + "tokio", +] + [[package]] name = "nix" version = "0.17.0" @@ -812,9 +837,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.22.3" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4916f159ed8e5de0082076562152a76b7a1f64a01fd9d1e0fea002c37624faf" +checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" dependencies = [ "bitflags", "cc", @@ -825,15 +850,13 @@ dependencies = [ [[package]] name = "nix" -version = "0.23.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" +checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" dependencies = [ "bitflags", - "cc", "cfg-if 1.0.0", "libc", - "memoffset", ] [[package]] @@ -1331,15 +1354,15 @@ dependencies = [ [[package]] name = "rtnetlink" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c9a6200d18ec1acfc218ce71363dcc9b6075f399220f903fdfeacd476a876ef" +checksum = "46f1cfa18f8cebe685373a2697915d7e0db3b4554918bba118385e0f71f258a7" dependencies = [ "futures", "log", "netlink-packet-route", "netlink-proto", - "nix 0.22.3", + "nix 0.24.2", "thiserror", "tokio", ] @@ -1710,20 +1733,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-util" -version = "0.6.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" -dependencies = [ - "bytes 1.1.0", - "futures-core", - "futures-sink", - "log", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-vsock" version = "0.3.1" diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index ae809bdaf7..a25120b4ff 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -32,7 +32,7 @@ tokio = { version = "1.14.0", features = ["full"] } tokio-vsock = "0.3.1" netlink-sys = { version = "0.7.0", features = ["tokio_socket",]} -rtnetlink = "0.8.0" +rtnetlink = "0.11.0" netlink-packet-utils = "0.4.1" ipnetwork = "0.17.0" diff --git a/src/agent/src/netlink.rs b/src/agent/src/netlink.rs index c6fc9c2079..171e2eb3b1 100644 --- a/src/agent/src/netlink.rs +++ b/src/agent/src/netlink.rs @@ -4,7 +4,7 @@ // use anyhow::{anyhow, Context, Result}; -use futures::{future, StreamExt, TryStreamExt}; +use futures::{future, TryStreamExt}; use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network}; use nix::errno::Errno; use protobuf::RepeatedField; @@ -164,7 +164,7 @@ impl Handle { let request = self.handle.link().get(); let filtered = match filter { - LinkFilter::Name(name) => request.set_name_filter(name.to_owned()), + LinkFilter::Name(name) => request.match_name(name.to_owned()), LinkFilter::Index(index) => request.match_index(index), _ => request, // Post filters }; @@ -516,7 +516,6 @@ impl Handle { } /// Adds an ARP neighbor. - /// TODO: `rtnetlink` has no neighbours API, remove this after https://github.com/little-dude/netlink/pull/135 async fn add_arp_neighbor(&mut self, neigh: &ARPNeighbor) -> Result<()> { let ip_address = neigh .toIPAddress @@ -528,58 +527,13 @@ impl Handle { let ip = IpAddr::from_str(ip_address) .map_err(|e| anyhow!("Failed to parse IP {}: {:?}", ip_address, e))?; - // Import rtnetlink objects that make sense only for this function - use packet::constants::{NDA_UNSPEC, NLM_F_ACK, NLM_F_CREATE, NLM_F_EXCL, NLM_F_REQUEST}; - use packet::neighbour::{NeighbourHeader, NeighbourMessage}; - use packet::nlas::neighbour::Nla; - use packet::{NetlinkMessage, NetlinkPayload, RtnlMessage}; - use rtnetlink::Error; - - const IFA_F_PERMANENT: u16 = 0x80; // See https://github.com/little-dude/netlink/blob/0185b2952505e271805902bf175fee6ea86c42b8/netlink-packet-route/src/rtnl/constants.rs#L770 - let link = self.find_link(LinkFilter::Name(&neigh.device)).await?; - let message = NeighbourMessage { - header: NeighbourHeader { - family: match ip { - IpAddr::V4(_) => packet::AF_INET, - IpAddr::V6(_) => packet::AF_INET6, - } as u8, - ifindex: link.index(), - state: if neigh.state != 0 { - neigh.state as u16 - } else { - IFA_F_PERMANENT - }, - flags: neigh.flags as u8, - ntype: NDA_UNSPEC as u8, - }, - nlas: { - let mut nlas = vec![Nla::Destination(match ip { - IpAddr::V4(v4) => v4.octets().to_vec(), - IpAddr::V6(v6) => v6.octets().to_vec(), - })]; - - if !neigh.lladdr.is_empty() { - nlas.push(Nla::LinkLocalAddress( - parse_mac_address(&neigh.lladdr)?.to_vec(), - )); - } - - nlas - }, - }; - - // Send request and ACK - let mut req = NetlinkMessage::from(RtnlMessage::NewNeighbour(message)); - req.header.flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE; - - let mut response = self.handle.request(req)?; - while let Some(message) = response.next().await { - if let NetlinkPayload::Error(err) = message.payload { - return Err(anyhow!(Error::NetlinkError(err))); - } - } + self.handle + .neighbours() + .add(link.index(), ip) + .execute() + .await?; Ok(()) } From 81fe51ab0b2873edc14d10c95593a65928889e73 Mon Sep 17 00:00:00 2001 From: Xuewei Niu Date: Mon, 1 Aug 2022 16:14:18 +0800 Subject: [PATCH 03/16] agent: fix unittests for arp neighbors Set an ARP address explicitly before netlink::test_add_one_arp_neighbor() running. Signed-off-by: Xuewei Niu --- src/agent/src/netlink.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/agent/src/netlink.rs b/src/agent/src/netlink.rs index 171e2eb3b1..3869eee000 100644 --- a/src/agent/src/netlink.rs +++ b/src/agent/src/netlink.rs @@ -904,7 +904,7 @@ mod tests { .expect("prepare: failed to delete neigh"); } - fn prepare_env_for_test_add_one_arp_neighbor(dummy_name: &str, ip: &str) { + fn prepare_env_for_test_add_one_arp_neighbor(dummy_name: &str, ip: &str, mac: &str) { clean_env_for_test_add_one_arp_neighbor(dummy_name, ip); // modprobe dummy Command::new("modprobe") @@ -918,6 +918,12 @@ mod tests { .output() .expect("failed to add dummy interface"); + // ip link set dummy address 6a:92:3a:59:70:aa + Command::new("ip") + .args(&["link", "set", dummy_name, "address", mac]) + .output() + .expect("failed to add dummy interface"); + // ip addr add 192.168.0.2/16 dev dummy Command::new("ip") .args(&["addr", "add", "192.168.0.2/16", "dev", dummy_name]) @@ -939,7 +945,7 @@ mod tests { let to_ip = "169.254.1.1"; let dummy_name = "dummy_for_arp"; - prepare_env_for_test_add_one_arp_neighbor(dummy_name, to_ip); + prepare_env_for_test_add_one_arp_neighbor(dummy_name, to_ip, mac); let mut ip_address = IPAddress::new(); ip_address.set_address(to_ip.to_string()); From 889557ecb128fa070598117300faeffe7e3dfe24 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Tue, 2 Aug 2022 15:20:45 +0800 Subject: [PATCH 04/16] docs: add back host network limitation Kata Containers doesn't support host network namespace, it's a common issue for new users. The limitation is deleted, this commit will add them back. Also, Docker has support to run containers using Kata Containers, delete Docker from not support list. This commit reverts parts of #3710 Fixes: #4794 Signed-off-by: Bin Liu --- docs/Limitations.md | 49 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/docs/Limitations.md b/docs/Limitations.md index 1c4cfcb7ba..d122e5ca50 100644 --- a/docs/Limitations.md +++ b/docs/Limitations.md @@ -60,17 +60,26 @@ This section lists items that might be possible to fix. ## OCI CLI commands ### Docker and Podman support -Currently Kata Containers does not support Docker or Podman. +Currently Kata Containers does not support Podman. See issue https://github.com/kata-containers/kata-containers/issues/722 for more information. +Docker supports Kata Containers since 22.06: + +```bash +$ sudo docker run --runtime io.containerd.kata.v2 +``` + +Kata Containers works perfectly with containerd, we recommend to use +containerd's Docker-style command line tool [`nerdctl`](https://github.com/containerd/nerdctl). + ## Runtime commands ### checkpoint and restore The runtime does not provide `checkpoint` and `restore` commands. There are discussions about using VM save and restore to give us a -`[criu](https://github.com/checkpoint-restore/criu)`-like functionality, +[`criu`](https://github.com/checkpoint-restore/criu)-like functionality, which might provide a solution. Note that the OCI standard does not specify `checkpoint` and `restore` @@ -93,6 +102,42 @@ All other configurations are supported and are working properly. ## Networking +### Host network + +Host network (`nerdctl/docker run --net=host`or [Kubernetes `HostNetwork`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#hosts-namespaces)) is not supported. +It is not possible to directly access the host networking configuration +from within the VM. + +The `--net=host` option can still be used with `runc` containers and +inter-mixed with running Kata Containers, thus enabling use of `--net=host` +when necessary. + +It should be noted, currently passing the `--net=host` option into a +Kata Container may result in the Kata Container networking setup +modifying, re-configuring and therefore possibly breaking the host +networking setup. Do not use `--net=host` with Kata Containers. + +### Support for joining an existing VM network + +Docker supports the ability for containers to join another containers +namespace with the `docker run --net=containers` syntax. This allows +multiple containers to share a common network namespace and the network +interfaces placed in the network namespace. Kata Containers does not +support network namespace sharing. If a Kata Container is setup to +share the network namespace of a `runc` container, the runtime +effectively takes over all the network interfaces assigned to the +namespace and binds them to the VM. Consequently, the `runc` container loses +its network connectivity. + +### docker run --link + +The runtime does not support the `docker run --link` command. This +command is now deprecated by docker and we have no intention of adding support. +Equivalent functionality can be achieved with the newer docker networking commands. + +See more documentation at +[docs.docker.com](https://docs.docker.com/network/links/). + ## Resource management Due to the way VMs differ in their CPU and memory allocation, and sharing From 230a229052c4619727a597738a82839cafb9ac61 Mon Sep 17 00:00:00 2001 From: Chen Yiyang Date: Thu, 28 Jul 2022 15:51:59 +0800 Subject: [PATCH 05/16] runk: add ps sub-command ps command supprot two formats, `json` and `table`. `json` format just outputs pids in the container. `table` format will use `ps` utilty in the host, search and output all processes in the container. Add a struct `container` to represent a spawned container. Move the `kill` implemention from kill.rs as a method of `container`. Fixes: #4361 Signed-off-by: Chen Yiyang --- src/tools/runk/libcontainer/src/builder.rs | 16 ++--- src/tools/runk/libcontainer/src/container.rs | 50 +++++++++++++++- src/tools/runk/src/commands/kill.rs | 32 ++-------- src/tools/runk/src/commands/mod.rs | 1 + src/tools/runk/src/commands/ps.rs | 63 ++++++++++++++++++++ src/tools/runk/src/main.rs | 1 + 6 files changed, 122 insertions(+), 41 deletions(-) create mode 100644 src/tools/runk/src/commands/ps.rs diff --git a/src/tools/runk/libcontainer/src/builder.rs b/src/tools/runk/libcontainer/src/builder.rs index 70bd2b3374..afb02b3565 100644 --- a/src/tools/runk/libcontainer/src/builder.rs +++ b/src/tools/runk/libcontainer/src/builder.rs @@ -503,11 +503,7 @@ mod tests { let root = tempdir().unwrap(); // let bundle = temp let id = "test".to_string(); - create_activated_dirs( - &root.path().to_path_buf(), - &id, - &bundle_dir.path().to_path_buf(), - ); + create_activated_dirs(root.path(), &id, bundle_dir.path()); let pid = getpid().as_raw(); let mut spec = create_dummy_spec(); @@ -517,7 +513,7 @@ mod tests { .to_string_lossy() .to_string(); - let status = create_dummy_status(&id, pid, &root.path().to_path_buf(), &spec); + let status = create_dummy_status(&id, pid, root.path(), &spec); status.save().unwrap(); let result = ActivatedContainerBuilder::default() @@ -600,13 +596,9 @@ mod tests { .join(TEST_ROOTFS_PATH) .to_string_lossy() .to_string(); - create_activated_dirs( - &root.path().to_path_buf(), - &id, - &bundle_dir.path().to_path_buf(), - ); + create_activated_dirs(root.path(), &id, bundle_dir.path()); - let status = create_dummy_status(&id, pid, &root.path().to_path_buf(), &spec); + let status = create_dummy_status(&id, pid, root.path(), &spec); status.save().unwrap(); let launcher = ActivatedContainerBuilder::default() .id(id) diff --git a/src/tools/runk/libcontainer/src/container.rs b/src/tools/runk/libcontainer/src/container.rs index ad6db7c817..abc40fbba3 100644 --- a/src/tools/runk/libcontainer/src/container.rs +++ b/src/tools/runk/libcontainer/src/container.rs @@ -3,9 +3,14 @@ // SPDX-License-Identifier: Apache-2.0 // -use crate::status::Status; +use crate::status::{self, get_all_pid, get_current_container_state, Status}; use anyhow::{anyhow, Result}; -use nix::unistd::{chdir, unlink}; +use nix::sys::signal::kill; +use nix::{ + sys::signal::Signal, + unistd::{chdir, unlink, Pid}, +}; +use oci::ContainerState; use rustjail::{ container::{BaseContainer, LinuxContainer, EXEC_FIFO_FILENAME}, process::{Process, ProcessOperations}, @@ -26,6 +31,47 @@ pub enum ContainerAction { Run, } +#[derive(Debug)] +pub struct Container { + pub status: Status, + pub state: ContainerState, +} + +impl Container { + pub fn load(state_root: &Path, id: &str) -> Result { + let status = Status::load(state_root, id)?; + let state = get_current_container_state(&status)?; + Ok(Self { status, state }) + } + + pub fn processes(&self) -> Result> { + get_all_pid(&self.status.cgroup_manager) + } + + pub fn kill(&self, signal: Signal, all: bool) -> Result<()> { + if all { + let pids = self.processes()?; + for pid in pids { + if !status::is_process_running(pid)? { + continue; + } + kill(pid, signal)?; + } + } else { + if self.state == ContainerState::Stopped { + return Err(anyhow!("container {} not running", self.status.id)); + } + let pid = Pid::from_raw(self.status.pid); + if status::is_process_running(pid)? { + kill(pid, signal)?; + } + } + Ok(()) + } + + // TODO: add pause and resume +} + /// Used to run a process. If init is set, it will create a container and run the process in it. /// If init is not set, it will run the process in an existing container. #[derive(Debug)] diff --git a/src/tools/runk/src/commands/kill.rs b/src/tools/runk/src/commands/kill.rs index 333f6b187a..0ae0d343dd 100644 --- a/src/tools/runk/src/commands/kill.rs +++ b/src/tools/runk/src/commands/kill.rs @@ -4,44 +4,22 @@ // use crate::Kill; -use anyhow::{anyhow, Result}; -use libcontainer::status::{self, get_current_container_state, Status}; -use nix::{ - sys::signal::{kill, Signal}, - unistd::Pid, -}; -use oci::ContainerState; +use anyhow::Result; +use libcontainer::container::Container; +use nix::sys::signal::Signal; use slog::{info, Logger}; use std::{convert::TryFrom, path::Path, str::FromStr}; pub fn run(opts: Kill, state_root: &Path, logger: &Logger) -> Result<()> { let container_id = &opts.container_id; - let status = Status::load(state_root, container_id)?; - let current_state = get_current_container_state(&status)?; + let container = Container::load(state_root, container_id)?; let sig = parse_signal(&opts.signal)?; // TODO: liboci-cli does not support --all option for kill command. // After liboci-cli supports the option, we will change the following code. // as a workaround we use a custom Kill command. let all = opts.all; - if all { - let pids = status::get_all_pid(&status.cgroup_manager)?; - for pid in pids { - if !status::is_process_running(pid)? { - continue; - } - kill(pid, sig)?; - } - } else { - if current_state == ContainerState::Stopped { - return Err(anyhow!("container {} not running", container_id)); - } - - let p = Pid::from_raw(status.pid); - if status::is_process_running(p)? { - kill(p, sig)?; - } - } + container.kill(sig, all)?; info!(&logger, "kill command finished successfully"); diff --git a/src/tools/runk/src/commands/mod.rs b/src/tools/runk/src/commands/mod.rs index e1e0810790..4243035214 100644 --- a/src/tools/runk/src/commands/mod.rs +++ b/src/tools/runk/src/commands/mod.rs @@ -8,6 +8,7 @@ pub mod delete; pub mod exec; pub mod kill; pub mod list; +pub mod ps; pub mod run; pub mod spec; pub mod start; diff --git a/src/tools/runk/src/commands/ps.rs b/src/tools/runk/src/commands/ps.rs new file mode 100644 index 0000000000..84bbc5ef2d --- /dev/null +++ b/src/tools/runk/src/commands/ps.rs @@ -0,0 +1,63 @@ +// Copyright 2021-2022 Kata Contributors +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::anyhow; +use anyhow::Result; +use libcontainer::container::Container; +use liboci_cli::Ps; +use slog::{info, Logger}; +use std::path::Path; +use std::process::Command; +use std::str; + +pub fn run(opts: Ps, root: &Path, logger: &Logger) -> Result<()> { + let container = Container::load(root, opts.container_id.as_str())?; + let pids = container + .processes()? + .iter() + .map(|pid| pid.as_raw()) + .collect::>(); + + match opts.format.as_str() { + "json" => println!("{}", serde_json::to_string(&pids)?), + "table" => { + let ps_options = if opts.ps_options.is_empty() { + vec!["-ef".to_string()] + } else { + opts.ps_options + }; + let output = Command::new("ps").args(ps_options).output()?; + if !output.status.success() { + return Err(anyhow!("{}", std::str::from_utf8(&output.stderr)?)); + } + let lines = str::from_utf8(&output.stdout)?.lines().collect::>(); + if lines.is_empty() { + return Err(anyhow!("no processes found")); + } + let pid_index = lines[0] + .split_whitespace() + .position(|field| field == "PID") + .ok_or_else(|| anyhow!("could't find PID field in ps output"))?; + println!("{}", lines[0]); + for &line in &lines[1..] { + if line.is_empty() { + continue; + } + let fields = line.split_whitespace().collect::>(); + if pid_index >= fields.len() { + continue; + } + let pid: i32 = fields[pid_index].parse()?; + if pids.contains(&pid) { + println!("{}", line); + } + } + } + _ => return Err(anyhow!("unknown format: {}", opts.format)), + } + + info!(&logger, "ps command finished successfully"); + Ok(()) +} diff --git a/src/tools/runk/src/main.rs b/src/tools/runk/src/main.rs index 6e5b976999..4565e6a369 100644 --- a/src/tools/runk/src/main.rs +++ b/src/tools/runk/src/main.rs @@ -80,6 +80,7 @@ async fn cmd_run(subcmd: SubCommand, root_path: &Path, logger: &Logger) -> Resul CommonCmd::Spec(spec) => commands::spec::run(spec, logger), CommonCmd::List(list) => commands::list::run(list, root_path, logger), CommonCmd::Exec(exec) => commands::exec::run(exec, root_path, logger).await, + CommonCmd::Ps(ps) => commands::ps::run(ps, root_path, logger), _ => { return Err(anyhow!("command is not implemented yet")); } From dd397ff1bf9518dfcf79459b121e88dbf4742c0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 3 Aug 2022 11:00:36 +0200 Subject: [PATCH 06/16] versions: Bump QEMU TDX version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's use the latest tag provided in the "https://github.com/intel/qemu-dcp" repo, "SPR-BKC-QEMU-v2.5". Fixes: #4802 Signed-off-by: Fabiano Fidêncio --- versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.yaml b/versions.yaml index 075b7e0070..d964cac7c0 100644 --- a/versions.yaml +++ b/versions.yaml @@ -101,7 +101,7 @@ assets: tdx: description: "VMM that uses KVM and supports TDX" url: "https://github.com/intel/qemu-dcp" - tag: "SPR-BKC-QEMU-v2.2" + tag: "SPR-BKC-QEMU-v2.5" qemu-experimental: description: "QEMU with virtiofs support" From c9358155a26cd2491431a2b96aba2c93b50ed8f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 3 Aug 2022 11:56:18 +0200 Subject: [PATCH 07/16] kernel: Sort the TDX configs alphabetically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's just re-order the TDX configs alphabetically. No new config has been added or removed, thus no need to bump the kernel version. Signed-off-by: Fabiano Fidêncio --- .../kernel/configs/fragments/x86_64/tdx/tdx.conf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/packaging/kernel/configs/fragments/x86_64/tdx/tdx.conf b/tools/packaging/kernel/configs/fragments/x86_64/tdx/tdx.conf index a363ec6b6e..9239aeecdc 100644 --- a/tools/packaging/kernel/configs/fragments/x86_64/tdx/tdx.conf +++ b/tools/packaging/kernel/configs/fragments/x86_64/tdx/tdx.conf @@ -1,13 +1,13 @@ # Intel Trust Domain Extensions (Intel TDX) +CONFIG_CLK_LGM_CGU=y +CONFIG_DMA_RESTRICTED_POOL=y CONFIG_EFI=y CONFIG_EFI_STUB=y -CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS=y -CONFIG_INTEL_TDX_GUEST=y CONFIG_INTEL_TDX_FIXES=y -CONFIG_X86_MEM_ENCRYPT_COMMON=y -CONFIG_X86_5LEVEL=y +CONFIG_INTEL_TDX_GUEST=y CONFIG_OF=y -CONFIG_CLK_LGM_CGU=y CONFIG_OF_RESERVED_MEM=y -CONFIG_DMA_RESTRICTED_POOL=y +CONFIG_X86_5LEVEL=y +CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS=y +CONFIG_X86_MEM_ENCRYPT_COMMON=y From 9972487f6e1a1ffd27c6f3e013d60efa4a3a2859 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 3 Aug 2022 11:58:34 +0200 Subject: [PATCH 08/16] versions: Bump Kernel TDX version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The latest kernel with TDX support should be pulled from a different repo (https://github.com/intel/linux-kernel-dcp, instead of https://github.com/intel/tdx), and the latest version to be used is SPR-BKC-PC-v9.6. With the new version being used, let's make sure we enable the INTEL_TDX_ATTESTATION config option, and all the dependencies needed to do so. Fixes: #4803 Signed-off-by: Fabiano Fidêncio --- tools/packaging/kernel/configs/fragments/x86_64/tdx/tdx.conf | 4 ++++ tools/packaging/kernel/kata_config_version | 2 +- versions.yaml | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/packaging/kernel/configs/fragments/x86_64/tdx/tdx.conf b/tools/packaging/kernel/configs/fragments/x86_64/tdx/tdx.conf index 9239aeecdc..1b1f8751ef 100644 --- a/tools/packaging/kernel/configs/fragments/x86_64/tdx/tdx.conf +++ b/tools/packaging/kernel/configs/fragments/x86_64/tdx/tdx.conf @@ -4,6 +4,8 @@ CONFIG_CLK_LGM_CGU=y CONFIG_DMA_RESTRICTED_POOL=y CONFIG_EFI=y CONFIG_EFI_STUB=y +CONFIG_INTEL_IOMMU_SVM=y +CONFIG_INTEL_TDX_ATTESTATION=y CONFIG_INTEL_TDX_FIXES=y CONFIG_INTEL_TDX_GUEST=y CONFIG_OF=y @@ -11,3 +13,5 @@ CONFIG_OF_RESERVED_MEM=y CONFIG_X86_5LEVEL=y CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS=y CONFIG_X86_MEM_ENCRYPT_COMMON=y +CONFIG_X86_PLATFORM_DEVICES=y +CONFIG_X86_PLATFORM_DRIVERS_INTEL=y diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version index c67f579c9a..49541f7210 100644 --- a/tools/packaging/kernel/kata_config_version +++ b/tools/packaging/kernel/kata_config_version @@ -1 +1 @@ -93 +94 diff --git a/versions.yaml b/versions.yaml index d964cac7c0..306e058e97 100644 --- a/versions.yaml +++ b/versions.yaml @@ -156,8 +156,8 @@ assets: version: "v5.15.48" tdx: description: "Linux kernel that supports TDX" - url: "https://github.com/intel/tdx/archive/refs/tags" - tag: "tdx-guest-v5.15-4" + url: "https://github.com/intel/linux-kernel-dcp/archive/refs/tags" + tag: "SPR-BKC-PC-v9.6" sev: description: "Linux kernel with efi_secret support" url: "https://github.com/confidential-containers-demo/\ From 62f05d4b481ab6765abe3de69628e6806770c02b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 3 Aug 2022 14:44:10 +0200 Subject: [PATCH 09/16] ovmf: Adjust final tarball location MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's create the OVMF tarball in the directory where the script was called from, instead of doing it in the $DESTDIR. This aligns with the logic being used for creating / extracting the tarball content, which is already in use by the kata-deploy local build scripts. Fixes: #4808 Signed-off-by: Fabiano Fidêncio --- tools/packaging/static-build/ovmf/build-ovmf.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/packaging/static-build/ovmf/build-ovmf.sh b/tools/packaging/static-build/ovmf/build-ovmf.sh index fe3925b1ce..9245dead48 100755 --- a/tools/packaging/static-build/ovmf/build-ovmf.sh +++ b/tools/packaging/static-build/ovmf/build-ovmf.sh @@ -90,7 +90,8 @@ if [ "${ovmf_build}" == "tdx" ]; then install $build_root/$ovmf_dir/"${build_path_arch}"/DumpTdxEventLog.efi ${install_dir} fi +local_dir=${PWD} pushd $DESTDIR -tar -czvf "${ovmf_dir}-${ovmf_build}.tar.gz" "./$PREFIX" +tar -czvf "${local_dir}/${ovmf_dir}-${ovmf_build}.tar.gz" "./$PREFIX" rm -rf $(dirname ./$PREFIX) popd From 8d1cb1d513e61ab9da47f397e1432c08fcb954e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 3 Aug 2022 14:47:03 +0200 Subject: [PATCH 10/16] td-shim: Adjust final tarball location MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's create the td-shim tarball in the directory where the script was called from, instead of doing it in the $DESTDIR. This aligns with the logic being used for creating / extracting the tarball content, which is already in use by the kata-deploy local build scripts. Fixes: #4809 Signed-off-by: Fabiano Fidêncio --- tools/packaging/static-build/td-shim/build-td-shim.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/packaging/static-build/td-shim/build-td-shim.sh b/tools/packaging/static-build/td-shim/build-td-shim.sh index ed933c007e..cbc336d666 100755 --- a/tools/packaging/static-build/td-shim/build-td-shim.sh +++ b/tools/packaging/static-build/td-shim/build-td-shim.sh @@ -35,7 +35,8 @@ install target/x86_64-unknown-uefi/release/final-boot-kernel.bin ${install_dir}/ popd #td-shim popd #${build_root} +local_dir=${PWD} pushd ${DESTDIR} -tar -czvf "td-shim.tar.gz" "./$PREFIX" +tar -czvf "${local_dir}/td-shim.tar.gz" "./$PREFIX" rm -rf $(dirname ./$PREFIX) popd #${DESTDIR} From c5452faec66cadf95c2a8efecd9f2f6944f7c41a Mon Sep 17 00:00:00 2001 From: Chelsea Mafrica Date: Thu, 4 Aug 2022 12:49:01 -0700 Subject: [PATCH 11/16] docs: Improve SGX documentation Update documentation with details regarding intel-device-plugins-for-kubernetes setup and dependencies. Fixes #4819 Signed-off-by: Chelsea Mafrica --- docs/use-cases/using-Intel-SGX-and-kata.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/use-cases/using-Intel-SGX-and-kata.md b/docs/use-cases/using-Intel-SGX-and-kata.md index f45e3ed5bc..b08e3c765c 100644 --- a/docs/use-cases/using-Intel-SGX-and-kata.md +++ b/docs/use-cases/using-Intel-SGX-and-kata.md @@ -18,7 +18,7 @@ CONFIG_X86_SGX_KVM=y * Kubernetes cluster configured with: * [`kata-deploy`](../../tools/packaging/kata-deploy) based Kata Containers installation - * [Intel SGX Kubernetes device plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/sgx_plugin#deploying-with-pre-built-images) + * [Intel SGX Kubernetes device plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/sgx_plugin#deploying-with-pre-built-images) and associated components including [operator](https://github.com/intel/intel-device-plugins-for-kubernetes/blob/main/cmd/operator/README.md) and dependencies > Note: Kata Containers supports creating VM sandboxes with Intel® SGX enabled > using [cloud-hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor/) and [QEMU](https://www.qemu.org/) VMMs only. @@ -99,4 +99,4 @@ because socket passthrough is not supported. An alternative is to deploy the `ae container. * Projects like [Gramine Shielded Containers (GSC)](https://gramine-gsc.readthedocs.io/en/latest/) are also known to work. For GSC specifically, the Kata guest kernel needs to have the `CONFIG_NUMA=y` -enabled and at least one CPU online when running the GSC container. +enabled and at least one CPU online when running the GSC container. The Kata Containers guest kernel currently has CONFIG_NUMA=y enabled by default. From d8ad16a34e9a785e5dcafdcc6b0ba52b361a9a4e Mon Sep 17 00:00:00 2001 From: chmod100 Date: Fri, 5 Aug 2022 11:36:40 +0000 Subject: [PATCH 12/16] runtime: add unlock before return in sendReq Unlock is required before return, so there need to add unlock Fixes: #4827 Signed-off-by: chmod100 --- src/runtime/virtcontainers/kata_agent.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index ba2fc811d2..f10f81daea 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -2015,11 +2015,13 @@ func (k *kataAgent) sendReq(spanCtx context.Context, request interface{}) (inter k.Lock() if k.reqHandlers == nil { + k.Unlock() return nil, errors.New("Client has already disconnected") } handler := k.reqHandlers[msgName] if msgName == "" || handler == nil { + k.Unlock() return nil, errors.New("Invalid request type") } From c8d4ea84e3d9da20e9a2ab8f8c35046792919da4 Mon Sep 17 00:00:00 2001 From: Chelsea Mafrica Date: Thu, 4 Aug 2022 14:27:08 -0700 Subject: [PATCH 13/16] docs: Improve SGX documentation Remove line about annotations support in CRI-O and containerd since it has been supported for a couple years. Fixes #4819 Signed-off-by: Chelsea Mafrica --- docs/use-cases/using-Intel-SGX-and-kata.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/use-cases/using-Intel-SGX-and-kata.md b/docs/use-cases/using-Intel-SGX-and-kata.md index b08e3c765c..9b721f1bc4 100644 --- a/docs/use-cases/using-Intel-SGX-and-kata.md +++ b/docs/use-cases/using-Intel-SGX-and-kata.md @@ -25,9 +25,6 @@ CONFIG_X86_SGX_KVM=y ### Kata Containers Configuration -Before running a Kata Container make sure that your version of `crio` or `containerd` -supports annotations. - For `containerd` check in `/etc/containerd/config.toml` that the list of `pod_annotations` passed to the `sandbox` are: `["io.katacontainers.*", "sgx.intel.com/epc"]`. @@ -99,4 +96,4 @@ because socket passthrough is not supported. An alternative is to deploy the `ae container. * Projects like [Gramine Shielded Containers (GSC)](https://gramine-gsc.readthedocs.io/en/latest/) are also known to work. For GSC specifically, the Kata guest kernel needs to have the `CONFIG_NUMA=y` -enabled and at least one CPU online when running the GSC container. The Kata Containers guest kernel currently has CONFIG_NUMA=y enabled by default. +enabled and at least one CPU online when running the GSC container. The Kata Containers guest kernel currently has `CONFIG_NUMA=y` enabled by default. From 1444d7ce42f97c37a63758180d1b9d543d8c00c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 8 Aug 2022 11:00:23 +0200 Subject: [PATCH 14/16] packaging: Create no_patches.txt for the SPR-BKC-PC-v9.6.x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The file was added as part of the commit that tested this changes in the CCv0 branch, but forgotten when re-writing it to the `main` branch. Fixes: #4841 Signed-off-by: Fabiano Fidêncio --- tools/packaging/kernel/patches/SPR-BKC-PC-v9.6.x/no_patches.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tools/packaging/kernel/patches/SPR-BKC-PC-v9.6.x/no_patches.txt diff --git a/tools/packaging/kernel/patches/SPR-BKC-PC-v9.6.x/no_patches.txt b/tools/packaging/kernel/patches/SPR-BKC-PC-v9.6.x/no_patches.txt new file mode 100644 index 0000000000..e69de29bb2 From 065305f4a12abaa2f23fb5ac91d8707be4968b9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 8 Aug 2022 11:21:38 +0200 Subject: [PATCH 15/16] agent-ctl: Add an empty [workspace] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "An empty [workspace] can be used with a package to conveniently create a workspace with the package and all of its path dependencies", according to the https://doc.rust-lang.org/cargo/reference/workspaces.html This is also matches with the suggestion provided by the Cargo itself, due to the errors faced with the Cloud Hypervisor CI: ``` 10:46:23 this may be fixable by adding `go/src/github.com/kata-containers/kata-containers/src/tools/agent-ctl` to the `workspace.members` array of the manifest located at: /tmp/jenkins/workspace/kata-containers-2-clh-PR/Cargo.toml 10:46:23 Alternatively, to keep it out of the workspace, add the package to the `workspace.exclude` array, or add an empty `[workspace]` table to the package's manifest. ``` Fixes: #4843 Signed-off-by: Fabiano Fidêncio --- src/tools/agent-ctl/Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tools/agent-ctl/Cargo.toml b/src/tools/agent-ctl/Cargo.toml index 8847f734f4..fba7dd41f5 100644 --- a/src/tools/agent-ctl/Cargo.toml +++ b/src/tools/agent-ctl/Cargo.toml @@ -39,3 +39,5 @@ humantime = "2.1.0" # For Options (state passing) serde = { version = "1.0.131", features = ["derive"] } serde_json = "1.0.73" + +[workspace] From 8a4e690089e901ed1082d7577d422849d189a257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 8 Aug 2022 15:50:11 +0200 Subject: [PATCH 16/16] versions: Update TD-shim due to build breakage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "We need a newer nightly 1.62 rust to deal with the change rust-lang/libc@576f778 on crate libc which breaks the compilation." This comes from the a pull-request raised on TD-shim repo, https://github.com/confidential-containers/td-shim/pull/354, which fixes the issues with the commit being used with Kata Containers. Let's bump to a newer commit of TD-shim and to a newer version of the nightly toolchain as part of our versions file. Fixes: #4840 Signed-off-by: Fabiano Fidêncio --- versions.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versions.yaml b/versions.yaml index 306e058e97..53fbe769d4 100644 --- a/versions.yaml +++ b/versions.yaml @@ -271,8 +271,8 @@ externals: td-shim: description: "Confidential Containers Shim Firmware" url: "https://github.com/confidential-containers/td-shim" - version: "5f62a0e367b1845a54e534d103ed4a697a599ac3" - toolchain: "nightly-2022-04-07" + version: "cf9592ef70bd6ba4c7ab1330d278a743f5ba3133" + toolchain: "nightly-2022-05-15" virtiofsd: description: "vhost-user virtio-fs device backend written in Rust"