From 59a05c7401ee358e5b70453eadc7f0ecaf335974 Mon Sep 17 00:00:00 2001 From: SinghWang Date: Mon, 20 Feb 2023 10:13:22 +0800 Subject: [PATCH 01/10] kata-deploy: Fix kata static firecracker arm64 package build error When building the kata static arm64 package, the stages of firecracker report errors. Fixes: #6318 Signed-off-by: SinghWang (cherry picked from commit 697ec8e578f32c238adf7c14262f6bf1437eadd7) Signed-off-by: Greg Kurz --- .../static-build/firecracker/build-static-firecracker.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/packaging/static-build/firecracker/build-static-firecracker.sh b/tools/packaging/static-build/firecracker/build-static-firecracker.sh index 2d98f4df70..1595754105 100755 --- a/tools/packaging/static-build/firecracker/build-static-firecracker.sh +++ b/tools/packaging/static-build/firecracker/build-static-firecracker.sh @@ -18,6 +18,8 @@ firecracker_repo="${firecracker_repo:-}" firecracker_dir="firecracker" firecracker_version="${firecracker_version:-}" +arch=$(uname -m) + if [ -z "$firecracker_repo" ]; then info "Get firecracker information from runtime versions.yaml" firecracker_url=$(get_from_kata_deps "assets.hypervisor.firecracker.url") @@ -37,5 +39,5 @@ git fetch git checkout ${firecracker_version} sudo ./tools/devtool --unattended build --release -ln -sf ./build/cargo_target/x86_64-unknown-linux-musl/release/firecracker ./firecracker-static -ln -sf ./build/cargo_target/x86_64-unknown-linux-musl/release/jailer ./jailer-static +ln -sf ./build/cargo_target/${arch}-unknown-linux-musl/release/firecracker ./firecracker-static +ln -sf ./build/cargo_target/${arch}-unknown-linux-musl/release/jailer ./jailer-static From d1305ee9eb5d61a9f8491454ace8a0af71a986ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Feb 2023 15:47:25 +0100 Subject: [PATCH 02/10] runtime-rs: Add a generic powerpc64le-options.mk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There's a check in the runtime-rs Makefile that basically checks whether the `arch/$arch-options.mk` exists or not and, if it doesn't, the build is just aborted. With this in mind, let's create a generic powerpc64le-options.mk file and not bail when building for this architecture. Fixes: #6142 Signed-off-by: Fabiano Fidêncio (cherry picked from commit be40683bc592f2373e3c282225e6605db8e202ac) Signed-off-by: Greg Kurz --- src/runtime-rs/arch/powerpc64le-options.mk | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/runtime-rs/arch/powerpc64le-options.mk diff --git a/src/runtime-rs/arch/powerpc64le-options.mk b/src/runtime-rs/arch/powerpc64le-options.mk new file mode 100644 index 0000000000..0a974680e0 --- /dev/null +++ b/src/runtime-rs/arch/powerpc64le-options.mk @@ -0,0 +1,15 @@ +# Copyright (c) 2019-2022 Alibaba Cloud +# Copyright (c) 2019-2022 Ant Group +# +# SPDX-License-Identifier: Apache-2.0 +# + +MACHINETYPE := pseries +KERNELPARAMS := +MACHINEACCELERATORS := "cap-cfpc=broken,cap-sbbc=broken,cap-ibs=broken,cap-large-decr=off,cap-ccf-assist=off" +CPUFEATURES := pmu=off + +QEMUCMD := qemu-system-ppc64 + +# dragonball binary name +DBCMD := dragonball From 592ecdb671472f85a222c0b1e7f7c2eae8a22cd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 16 Feb 2023 09:12:05 +0100 Subject: [PATCH 03/10] packaging/shim-v2: Install the target depending on the arch/libc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the `install_go_rust.sh` file we're adding a x86_64-unknown-linux-musl target unconditionally. That should be, instead, based in the ARCH of the host and the appropriate LIBC to be used with that host. Signed-off-by: Fabiano Fidêncio (cherry picked from commit 47c058599a3980e3d4dd86b1282561f86646dc55) Signed-off-by: Greg Kurz --- .../static-build/shim-v2/install_go_rust.sh | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/tools/packaging/static-build/shim-v2/install_go_rust.sh b/tools/packaging/static-build/shim-v2/install_go_rust.sh index db192f673b..502d5f085c 100755 --- a/tools/packaging/static-build/shim-v2/install_go_rust.sh +++ b/tools/packaging/static-build/shim-v2/install_go_rust.sh @@ -50,12 +50,37 @@ EOF trap finish EXIT +go_version=${1:-} rust_version=${2:-} + ARCH=${ARCH:-$(uname -m)} -LIBC=${LIBC:-musl} +case "${ARCH}" in + aarch64) + goarch=arm64 + LIBC=musl + ;; + ppc64le) + goarch=${ARCH} + ARCH=powerpc64le + LIBC=gnu + ;; + s390x) + goarch=${ARCH} + LIBC=gnu + ;; + x86_64) + goarch=amd64 + LIBC=musl + ;; + *) + echo "unsupported architecture $(uname -m)" + exit 1 + ;; +esac + curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSLf | sh -s -- -y --default-toolchain ${rust_version} -t ${ARCH}-unknown-linux-${LIBC} source /root/.cargo/env -rustup target add x86_64-unknown-linux-musl +rustup target add ${ARCH}-unknown-linux-${LIBC} pushd "${tmp_dir}" @@ -70,9 +95,6 @@ done shift $(( $OPTIND - 1 )) - -go_version=${1:-} - if [ -z "$go_version" ];then echo "Missing go" usage 1 @@ -90,14 +112,6 @@ if command -v go; then fi fi -case "$(uname -m)" in - aarch64) goarch="arm64";; - ppc64le) goarch="ppc64le";; - x86_64) goarch="amd64";; - s390x) goarch="s390x";; - *) echo "unsupported architecture: $(uname -m)"; exit 1;; -esac - info "Download go version ${go_version}" kernel_name=$(uname -s) curl -OL "https://storage.googleapis.com/golang/go${go_version}.${kernel_name,,}-${goarch}.tar.gz" From 697707493084b3aa40cface2c01079282d80a652 Mon Sep 17 00:00:00 2001 From: SinghWang Date: Mon, 20 Feb 2023 13:45:11 +0800 Subject: [PATCH 04/10] kata-deploy: Fix static shim-v2 build on arm64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Following Jong Wu suggestion, let's link /usr/bin/musl-gcc to /usr/bin/aarch64-linux-musl-gcc. Fixes: #6320 Signed-off-by: SinghWang Signed-off-by: Fabiano Fidêncio (cherry picked from commit b4a1527aa664ec54ce4fc2941bc5858622ba944f) Signed-off-by: Greg Kurz --- tools/packaging/static-build/shim-v2/install_go_rust.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/packaging/static-build/shim-v2/install_go_rust.sh b/tools/packaging/static-build/shim-v2/install_go_rust.sh index 502d5f085c..078f49d37d 100755 --- a/tools/packaging/static-build/shim-v2/install_go_rust.sh +++ b/tools/packaging/static-build/shim-v2/install_go_rust.sh @@ -58,6 +58,10 @@ case "${ARCH}" in aarch64) goarch=arm64 LIBC=musl + # This is a hack needed as part of Ubuntu 20.04 + if [ ! -f /usr/bin/aarch64-linux-musl-gcc ]; then + ln -sf /usr/bin/musl-gcc /usr/bin/aarch64-linux-musl-gcc + fi ;; ppc64le) goarch=${ARCH} From fcab7c3a01b4f9e27ee018698c4b9321b135730c Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Fri, 10 Feb 2023 11:43:31 +0100 Subject: [PATCH 05/10] osbuilder: Include minimal set of device nodes in ubuntu initrd When starting an initrd the kernel expects to find /dev/console in the initrd, so that it can connect it as stdin/stdout/stderr to the /init process. If the device node is missing the kernel will complain that it was unable to open an initial console. If kata-agent is the initrd init process, it will also result in log messages not being logged to console and thus not forwarded to host syslog. Add a set of standard device nodes for completeness, so that console logging works. To do that we install the makedev packge which provides a MAKEDEV helper that knows the major/minor numbers. Unfortunately the debian package tries to create devnodes from postinst, which can be suppressed if systemd-detect-virt is present. That's why we create a small dummy script that matches what systemd-detect-virt would output (anything is enough to suppress mknod). Fixes: #6261 Signed-off-by: Jeremi Piotrowski (cherry picked from commit 76e926453a02bf52d62c9de65c549d7828ae4d65) Signed-off-by: Greg Kurz --- tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in | 3 +++ tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in index de3c31ed93..937ea89f43 100644 --- a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in +++ b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in @@ -6,6 +6,8 @@ ARG IMAGE_REGISTRY=docker.io FROM ${IMAGE_REGISTRY}/ubuntu:@OS_VERSION@ @SET_PROXY@ +# makedev tries to mknod from postinst +RUN [ -x /usr/bin/systemd-detect-virt ] || ( echo "echo docker" >/usr/bin/systemd-detect-virt && chmod +x /usr/bin/systemd-detect-virt ) RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive \ apt-get --no-install-recommends -y install \ @@ -20,6 +22,7 @@ RUN apt-get update && \ echo "gcc-$gcc_arch-linux-gnu libc6-dev-$libc_arch-cross")) \ git \ make \ + makedev \ multistrap \ musl-tools \ protobuf-compiler diff --git a/tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh b/tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh index daa158bf70..b53749edc5 100644 --- a/tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh +++ b/tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh @@ -34,4 +34,10 @@ EOF # Reduce image size and memory footprint by removing unnecessary files and directories. rm -rf $rootfs_dir/usr/share/{bash-completion,bug,doc,info,lintian,locale,man,menu,misc,pixmaps,terminfo,zsh} + + # Minimal set of device nodes needed when AGENT_INIT=yes so that the + # kernel can properly setup stdout/stdin/stderr for us + pushd $rootfs_dir/dev + MAKEDEV -v console tty ttyS null zero fd + popd } From 624dc2d222222f69ee302dc01a3bf216df69819c Mon Sep 17 00:00:00 2001 From: XDTG Date: Fri, 24 Feb 2023 15:48:09 +0800 Subject: [PATCH 06/10] runtime: use filepath.Clean() to clean the mount path Fix path check bypassed issuse introduced by #6082, use filepath.Clean() to clean path before check Fixes: #6082 Signed-off-by: XDTG (cherry picked from commit dc86d6dac35f17911ebeaab9a612173b005b7ac8) Signed-off-by: Greg Kurz --- src/runtime/virtcontainers/mount.go | 2 ++ src/runtime/virtcontainers/mount_linux_test.go | 3 +++ src/runtime/virtcontainers/mount_test.go | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/src/runtime/virtcontainers/mount.go b/src/runtime/virtcontainers/mount.go index 243c13f330..6c2e204208 100644 --- a/src/runtime/virtcontainers/mount.go +++ b/src/runtime/virtcontainers/mount.go @@ -44,6 +44,7 @@ func mountLogger() *logrus.Entry { } func isSystemMount(m string) bool { + m = filepath.Clean(m) for _, p := range systemMountPrefixes { if m == p || strings.HasPrefix(m, p+"/") { return true @@ -54,6 +55,7 @@ func isSystemMount(m string) bool { } func isHostDevice(m string) bool { + m = filepath.Clean(m) if m == "/dev" { return true } diff --git a/src/runtime/virtcontainers/mount_linux_test.go b/src/runtime/virtcontainers/mount_linux_test.go index a34f7c28f3..e5019b401b 100644 --- a/src/runtime/virtcontainers/mount_linux_test.go +++ b/src/runtime/virtcontainers/mount_linux_test.go @@ -249,6 +249,9 @@ func TestIsHostDevice(t *testing.T) { {"/dev/zero", true}, {"/dev/block", true}, {"/mnt/dev/block", false}, + {"/../dev", true}, + {"/../dev/block", true}, + {"/../mnt/dev/block", false}, } for _, test := range tests { diff --git a/src/runtime/virtcontainers/mount_test.go b/src/runtime/virtcontainers/mount_test.go index 6d91d22a7b..c21d00a195 100644 --- a/src/runtime/virtcontainers/mount_test.go +++ b/src/runtime/virtcontainers/mount_test.go @@ -41,6 +41,10 @@ func TestIsSystemMount(t *testing.T) { {"/home", false}, {"/dev/block/", false}, {"/mnt/dev/foo", false}, + {"/../sys", true}, + {"/../sys/", true}, + {"/../sys/fs/cgroup", true}, + {"/../sysfoo", false}, } for _, test := range tests { From 491b95451c6f0c7d6919242c730c9213109d09c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 28 Feb 2023 23:48:37 +0100 Subject: [PATCH 07/10] workflows: Do not install docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The latest ubuntu runners already have docker installed and trying to install it manually will cause the following issue: ``` Run curl -fsSL https://test.docker.com/ -o test-docker.sh Warning: the "docker" command appears to already exist on this system. If you already have Docker installed, this script can cause trouble, which is why we're displaying this warning and provide the opportunity to cancel the installation. If you installed the current Docker package using this script and are using it again to update Docker, you can safely ignore this message. You may press Ctrl+C now to abort this script. + sleep 20 + sudo -E sh -c apt-get update -qq >/dev/null E: The repository 'https://packages.microsoft.com/ubuntu/22.04/prod jammy Release' is no longer signed. ``` Fixes: #6390 Signed-off-by: Fabiano Fidêncio (cherry picked from commit 828d467222d4c1b161dcf8bf9acba0634e7ccf08) Signed-off-by: Greg Kurz --- .github/workflows/kata-deploy-push.yaml | 6 ------ .github/workflows/kata-deploy-test.yaml | 5 ----- .github/workflows/release.yaml | 5 ----- 3 files changed, 16 deletions(-) diff --git a/.github/workflows/kata-deploy-push.yaml b/.github/workflows/kata-deploy-push.yaml index c7d7e8cb4e..ce45ab5abe 100644 --- a/.github/workflows/kata-deploy-push.yaml +++ b/.github/workflows/kata-deploy-push.yaml @@ -29,12 +29,6 @@ jobs: - nydus steps: - uses: actions/checkout@v2 - - name: Install docker - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - curl -fsSL https://test.docker.com -o test-docker.sh - sh test-docker.sh - - name: Build ${{ matrix.asset }} if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} run: | diff --git a/.github/workflows/kata-deploy-test.yaml b/.github/workflows/kata-deploy-test.yaml index 5a924c7390..31cdaf9c8f 100644 --- a/.github/workflows/kata-deploy-test.yaml +++ b/.github/workflows/kata-deploy-test.yaml @@ -72,11 +72,6 @@ jobs: with: ref: ${{ steps.get-PR-ref.outputs.pr-ref }} - - name: Install docker - run: | - curl -fsSL https://test.docker.com -o test-docker.sh - sh test-docker.sh - - name: Build ${{ matrix.asset }} run: | make "${KATA_ASSET}-tarball" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index bbb95c53da..74c43bfbb3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -22,11 +22,6 @@ jobs: - virtiofsd steps: - uses: actions/checkout@v2 - - name: Install docker - run: | - curl -fsSL https://test.docker.com -o test-docker.sh - sh test-docker.sh - - name: Build ${{ matrix.asset }} run: | ./tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh From 12ec33d70de766e6fbd686beeea311f1262e1f06 Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Fri, 10 Feb 2023 14:34:53 +0100 Subject: [PATCH 08/10] rustjail: print type of cgroup manager Since the cgroup manager is wrapped in a dyn now, the print in LinuxContainer::new has been useless and just says "CgroupManager". Extend the Debug trait for 'dyn Manager' to print the type of the cgroup manager so that it's easier to debug issues. Fixes: #5779 Signed-off-by: Jeremi Piotrowski (cherry picked from commit ad8968c8d99a6089a0fdcfae48eb55e6fd1a0504) Signed-off-by: Greg Kurz --- src/agent/rustjail/src/cgroups/fs/mod.rs | 4 ++++ src/agent/rustjail/src/cgroups/mock.rs | 4 ++++ src/agent/rustjail/src/cgroups/mod.rs | 4 +++- src/agent/rustjail/src/cgroups/systemd/manager.rs | 4 ++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs index becc560363..6eaa9870df 100644 --- a/src/agent/rustjail/src/cgroups/fs/mod.rs +++ b/src/agent/rustjail/src/cgroups/fs/mod.rs @@ -267,6 +267,10 @@ impl CgroupManager for Manager { fn as_any(&self) -> Result<&dyn Any> { Ok(self) } + + fn name(&self) -> &str { + "cgroupfs" + } } fn set_network_resources( diff --git a/src/agent/rustjail/src/cgroups/mock.rs b/src/agent/rustjail/src/cgroups/mock.rs index fbad6d6642..3bcc99955c 100644 --- a/src/agent/rustjail/src/cgroups/mock.rs +++ b/src/agent/rustjail/src/cgroups/mock.rs @@ -66,6 +66,10 @@ impl CgroupManager for Manager { fn as_any(&self) -> Result<&dyn Any> { Ok(self) } + + fn name(&self) -> &str { + "mock" + } } impl Manager { diff --git a/src/agent/rustjail/src/cgroups/mod.rs b/src/agent/rustjail/src/cgroups/mod.rs index 24275c3148..c4e3b178b5 100644 --- a/src/agent/rustjail/src/cgroups/mod.rs +++ b/src/agent/rustjail/src/cgroups/mod.rs @@ -52,10 +52,12 @@ pub trait Manager { fn as_any(&self) -> Result<&dyn Any> { Err(anyhow!("not supported!")) } + + fn name(&self) -> &str; } impl Debug for dyn Manager + Send + Sync { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "CgroupManager") + write!(f, "{}", self.name()) } } diff --git a/src/agent/rustjail/src/cgroups/systemd/manager.rs b/src/agent/rustjail/src/cgroups/systemd/manager.rs index 481ef1e60e..c52e727e4c 100644 --- a/src/agent/rustjail/src/cgroups/systemd/manager.rs +++ b/src/agent/rustjail/src/cgroups/systemd/manager.rs @@ -101,6 +101,10 @@ impl CgroupManager for Manager { fn as_any(&self) -> Result<&dyn Any> { Ok(self) } + + fn name(&self) -> &str { + "systemd" + } } impl Manager { From be512e7f346a3550d52bbc3421d2f531b2c65f70 Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Fri, 24 Feb 2023 13:43:26 +0100 Subject: [PATCH 09/10] agent: determine value of use_systemd_cgroup before LinuxContainer::new() Right now LinuxContainer::new() gets passed a CreateOpts struct, but then modifies the use_systemd_cgroup field inside that struct. Pull the cgroups path parsing logic into do_create_container, so that CreateOpts can be immutable in LinuxContainer::new. This is just moving things around, there should be no functional changes. Signed-off-by: Jeremi Piotrowski (cherry picked from commit b0691806f1437951c38436e28aa086ae9c6f00d9) Signed-off-by: Greg Kurz --- src/agent/rustjail/src/container.rs | 11 ++--------- src/agent/src/rpc.rs | 10 ++++++++-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index f75b6e6d54..ed69ef60d1 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -1449,7 +1449,7 @@ impl LinuxContainer { pub fn new + Display + Clone>( id: T, base: T, - mut config: Config, + config: Config, logger: &Logger, ) -> Result { let base = base.into(); @@ -1475,21 +1475,14 @@ impl LinuxContainer { .context(format!("Cannot change owner of container {} root", id))?; let spec = config.spec.as_ref().unwrap(); - let linux = spec.linux.as_ref().unwrap(); - - // determine which cgroup driver to take and then assign to config.use_systemd_cgroup - // systemd: "[slice]:[prefix]:[name]" - // fs: "/path_a/path_b" - let cpath = if SYSTEMD_CGROUP_PATH_FORMAT.is_match(linux.cgroups_path.as_str()) { - config.use_systemd_cgroup = true; + let cpath = if config.use_systemd_cgroup { if linux.cgroups_path.len() == 2 { format!("system.slice:kata_agent:{}", id.as_str()) } else { linux.cgroups_path.clone() } } else { - config.use_systemd_cgroup = false; if linux.cgroups_path.is_empty() { format!("/{}", id.as_str()) } else { diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index c52d866d65..0081765416 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -36,7 +36,7 @@ use protocols::health::{ use protocols::types::Interface; use protocols::{agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc}; use rustjail::cgroups::notifier; -use rustjail::container::{BaseContainer, Container, LinuxContainer}; +use rustjail::container::{BaseContainer, Container, LinuxContainer, SYSTEMD_CGROUP_PATH_FORMAT}; use rustjail::process::Process; use rustjail::specconv::CreateOpts; @@ -210,9 +210,15 @@ impl AgentService { // restore the cwd for kata-agent process. defer!(unistd::chdir(&olddir).unwrap()); + // determine which cgroup driver to take and then assign to use_systemd_cgroup + // systemd: "[slice]:[prefix]:[name]" + // fs: "/path_a/path_b" + let cgroups_path = oci.linux.as_ref().map_or("", |linux| &linux.cgroups_path); + let use_systemd_cgroup = SYSTEMD_CGROUP_PATH_FORMAT.is_match(cgroups_path); + let opts = CreateOpts { cgroup_name: "".to_string(), - use_systemd_cgroup: false, + use_systemd_cgroup, no_pivot_root: s.no_pivot_root, no_new_keyring: false, spec: Some(oci.clone()), From 3eb7387bb741cc16a126a99f267e4dd078692a95 Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Fri, 10 Feb 2023 14:50:54 +0100 Subject: [PATCH 10/10] agent: always use cgroupfs when running as init The logic to decide which cgroup driver is used is currently based on the cgroup path that the host provides. This requires host and guest to use the same cgroup driver. If the guest uses kata-agent as init, then systemd can't be used as the cgroup driver. If the host requests a systemd cgroup, this currently results in a rustjail panic: thread 'tokio-runtime-worker' panicked at 'called `Result::unwrap()` on an `Err` value: I/O error: No such file or directory (os error 2) Caused by: No such file or directory (os error 2)', rustjail/src/cgroups/systemd/manager.rs:44:51 stack backtrace: 0: 0x7ff0fe77a793 - std::backtrace_rs::backtrace::libunwind::trace::h8c197fa9a679d134 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5 1: 0x7ff0fe77a793 - std::backtrace_rs::backtrace::trace_unsynchronized::h9ee19d58b6d5934a at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5 2: 0x7ff0fe77a793 - std::sys_common::backtrace::_print_fmt::h4badc450600fc417 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys_common/backtrace.rs:65:5 3: 0x7ff0fe77a793 - ::fmt::had334ddb529a2169 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys_common/backtrace.rs:44:22 4: 0x7ff0fdce815e - core::fmt::write::h1aa7694f03e44db2 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/core/src/fmt/mod.rs:1209:17 5: 0x7ff0fe74e0c4 - std::io::Write::write_fmt::h61b2bdc565be41b5 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/io/mod.rs:1682:15 6: 0x7ff0fe77cd3f - std::sys_common::backtrace::_print::h4ec69798b72ff254 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys_common/backtrace.rs:47:5 7: 0x7ff0fe77cd3f - std::sys_common::backtrace::print::h0e6c02048dec3c77 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys_common/backtrace.rs:34:9 8: 0x7ff0fe77c93f - std::panicking::default_hook::{{closure}}::hcdb7e705dc37ea6e at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/panicking.rs:267:22 9: 0x7ff0fe77d9b8 - std::panicking::default_hook::he03a933a0f01790f at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/panicking.rs:286:9 10: 0x7ff0fe77d9b8 - std::panicking::rust_panic_with_hook::he26b680bfd953008 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/panicking.rs:688:13 11: 0x7ff0fe77d482 - std::panicking::begin_panic_handler::{{closure}}::h559120d2dd1c6180 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/panicking.rs:579:13 12: 0x7ff0fe77d3ec - std::sys_common::backtrace::__rust_end_short_backtrace::h36db621fc93b005a at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys_common/backtrace.rs:137:18 13: 0x7ff0fe77d3c1 - rust_begin_unwind at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/panicking.rs:575:5 14: 0x7ff0fda52ee2 - core::panicking::panic_fmt::he7679b415d25c5f4 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/core/src/panicking.rs:65:14 15: 0x7ff0fda53182 - core::result::unwrap_failed::hb71caff146724b6b at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/core/src/result.rs:1791:5 16: 0x7ff0fe5bd738 - ::apply::hd46958d9d807d2ca 17: 0x7ff0fe606d80 - ::start::{{closure}}::h1de806d91fcb878f 18: 0x7ff0fe604a76 - as core::future::future::Future>::poll::h1749c148adcc235f 19: 0x7ff0fdc0c992 - kata_agent::rpc::AgentService::do_create_container::{{closure}}::{{closure}}::hc1b87a15dfdf2f64 20: 0x7ff0fdb80ae4 - as core::future::future::Future>::poll::h846a8c9e4fb67707 21: 0x7ff0fe3bb816 - as core::future::future::Future>::poll::h53de16ff66ed3972 22: 0x7ff0fdb519cb - as core::future::future::Future>::poll::h1cbece980286c0f4 23: 0x7ff0fdf4019c - as core::future::future::Future>::poll::hc8e72d155feb8d1f 24: 0x7ff0fdfa5fd8 - tokio::loom::std::unsafe_cell::UnsafeCell::with_mut::h0a407ffe2559449a 25: 0x7ff0fdf033a1 - tokio::runtime::task::raw::poll::h1045d9f1db9742de 26: 0x7ff0fe7a8ce2 - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::h4924ae3464af7fbd 27: 0x7ff0fe7afb85 - tokio::runtime::task::raw::poll::h5c843be39646b833 28: 0x7ff0fe7a05ee - std::sys_common::backtrace::__rust_begin_short_backtrace::ha7777c55b98a9bd1 29: 0x7ff0fe7a9bdb - core::ops::function::FnOnce::call_once{{vtable.shim}}::h27ec83c953360cdd 30: 0x7ff0fe7801d5 - as core::ops::function::FnOnce>::call_once::hed812350c5aef7a8 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/alloc/src/boxed.rs:1987:9 31: 0x7ff0fe7801d5 - as core::ops::function::FnOnce>::call_once::hc7df8e435a658960 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/alloc/src/boxed.rs:1987:9 32: 0x7ff0fe7801d5 - std::sys::unix::thread::Thread::new::thread_start::h575491a8a17dbb33 at /rustc/69f9c33d71c871fc16ac445211281c6e7a340943/library/std/src/sys/unix/thread.rs:108:17 Forward the value of "init_mode" to AgentService, so that we can force cgroupfs when systemd is unavailable. Fixes: #5779 Signed-off-by: Jeremi Piotrowski (cherry picked from commit 192df845885f5dab27abf78751b3035f68e3d786) Signed-off-by: Greg Kurz --- src/agent/rustjail/src/container.rs | 9 ++++----- src/agent/src/main.rs | 2 +- src/agent/src/rpc.rs | 21 +++++++++++++++++---- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index ed69ef60d1..15b7d39cef 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -1482,12 +1482,11 @@ impl LinuxContainer { } else { linux.cgroups_path.clone() } + } else if linux.cgroups_path.is_empty() { + format!("/{}", id.as_str()) } else { - if linux.cgroups_path.is_empty() { - format!("/{}", id.as_str()) - } else { - linux.cgroups_path.clone() - } + // if we have a systemd cgroup path we need to convert it to a fs cgroup path + linux.cgroups_path.replace(':', "/") }; let cgroup_manager: Box = if config.use_systemd_cgroup { diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 2c1f41defe..d8e9fc828b 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -339,7 +339,7 @@ async fn start_sandbox( sandbox.lock().await.sender = Some(tx); // vsock:///dev/vsock, port - let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str())?; + let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode)?; server.start().await?; rx.await?; diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 0081765416..3be8fed5c8 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -137,6 +137,7 @@ macro_rules! is_allowed { #[derive(Clone, Debug)] pub struct AgentService { sandbox: Arc>, + init_mode: bool, } impl AgentService { @@ -213,8 +214,13 @@ impl AgentService { // determine which cgroup driver to take and then assign to use_systemd_cgroup // systemd: "[slice]:[prefix]:[name]" // fs: "/path_a/path_b" + // If agent is init we can't use systemd cgroup mode, no matter what the host tells us let cgroups_path = oci.linux.as_ref().map_or("", |linux| &linux.cgroups_path); - let use_systemd_cgroup = SYSTEMD_CGROUP_PATH_FORMAT.is_match(cgroups_path); + let use_systemd_cgroup = if self.init_mode { + false + } else { + SYSTEMD_CGROUP_PATH_FORMAT.is_match(cgroups_path) + }; let opts = CreateOpts { cgroup_name: "".to_string(), @@ -1679,9 +1685,11 @@ async fn read_stream(reader: Arc>>, l: usize) -> Resu Ok(content) } -pub fn start(s: Arc>, server_address: &str) -> Result { - let agent_service = - Box::new(AgentService { sandbox: s }) as Box; +pub fn start(s: Arc>, server_address: &str, init_mode: bool) -> Result { + let agent_service = Box::new(AgentService { + sandbox: s, + init_mode, + }) as Box; let agent_worker = Arc::new(agent_service); @@ -2157,6 +2165,7 @@ mod tests { let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), + init_mode: true, }); let req = protocols::agent::UpdateInterfaceRequest::default(); @@ -2174,6 +2183,7 @@ mod tests { let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), + init_mode: true, }); let req = protocols::agent::UpdateRoutesRequest::default(); @@ -2191,6 +2201,7 @@ mod tests { let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), + init_mode: true, }); let req = protocols::agent::AddARPNeighborsRequest::default(); @@ -2324,6 +2335,7 @@ mod tests { let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), + init_mode: true, }); let result = agent_service @@ -2804,6 +2816,7 @@ OtherField:other let sandbox = Sandbox::new(&logger).unwrap(); let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), + init_mode: true, }); let ctx = mk_ttrpc_context();