diff --git a/.github/workflows/kata-deploy-push.yaml b/.github/workflows/kata-deploy-push.yaml index c7d7e8cb4e..ce45ab5abe 100644 --- a/.github/workflows/kata-deploy-push.yaml +++ b/.github/workflows/kata-deploy-push.yaml @@ -29,12 +29,6 @@ jobs: - nydus steps: - uses: actions/checkout@v2 - - name: Install docker - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - curl -fsSL https://test.docker.com -o test-docker.sh - sh test-docker.sh - - name: Build ${{ matrix.asset }} if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} run: | diff --git a/.github/workflows/kata-deploy-test.yaml b/.github/workflows/kata-deploy-test.yaml index 5a924c7390..31cdaf9c8f 100644 --- a/.github/workflows/kata-deploy-test.yaml +++ b/.github/workflows/kata-deploy-test.yaml @@ -72,11 +72,6 @@ jobs: with: ref: ${{ steps.get-PR-ref.outputs.pr-ref }} - - name: Install docker - run: | - curl -fsSL https://test.docker.com -o test-docker.sh - sh test-docker.sh - - name: Build ${{ matrix.asset }} run: | make "${KATA_ASSET}-tarball" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index bbb95c53da..74c43bfbb3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -22,11 +22,6 @@ jobs: - virtiofsd steps: - uses: actions/checkout@v2 - - name: Install docker - run: | - curl -fsSL https://test.docker.com -o test-docker.sh - sh test-docker.sh - - name: Build ${{ matrix.asset }} run: | ./tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs index becc560363..6eaa9870df 100644 --- a/src/agent/rustjail/src/cgroups/fs/mod.rs +++ b/src/agent/rustjail/src/cgroups/fs/mod.rs @@ -267,6 +267,10 @@ impl CgroupManager for Manager { fn as_any(&self) -> Result<&dyn Any> { Ok(self) } + + fn name(&self) -> &str { + "cgroupfs" + } } fn set_network_resources( diff --git a/src/agent/rustjail/src/cgroups/mock.rs b/src/agent/rustjail/src/cgroups/mock.rs index fbad6d6642..3bcc99955c 100644 --- a/src/agent/rustjail/src/cgroups/mock.rs +++ b/src/agent/rustjail/src/cgroups/mock.rs @@ -66,6 +66,10 @@ impl CgroupManager for Manager { fn as_any(&self) -> Result<&dyn Any> { Ok(self) } + + fn name(&self) -> &str { + "mock" + } } impl Manager { diff --git a/src/agent/rustjail/src/cgroups/mod.rs b/src/agent/rustjail/src/cgroups/mod.rs index 24275c3148..c4e3b178b5 100644 --- a/src/agent/rustjail/src/cgroups/mod.rs +++ b/src/agent/rustjail/src/cgroups/mod.rs @@ -52,10 +52,12 @@ pub trait Manager { fn as_any(&self) -> Result<&dyn Any> { Err(anyhow!("not supported!")) } + + fn name(&self) -> &str; } impl Debug for dyn Manager + Send + Sync { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "CgroupManager") + write!(f, "{}", self.name()) } } diff --git a/src/agent/rustjail/src/cgroups/systemd/manager.rs b/src/agent/rustjail/src/cgroups/systemd/manager.rs index 481ef1e60e..c52e727e4c 100644 --- a/src/agent/rustjail/src/cgroups/systemd/manager.rs +++ b/src/agent/rustjail/src/cgroups/systemd/manager.rs @@ -101,6 +101,10 @@ impl CgroupManager for Manager { fn as_any(&self) -> Result<&dyn Any> { Ok(self) } + + fn name(&self) -> &str { + "systemd" + } } impl Manager { diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs index f75b6e6d54..15b7d39cef 100644 --- a/src/agent/rustjail/src/container.rs +++ b/src/agent/rustjail/src/container.rs @@ -1449,7 +1449,7 @@ impl LinuxContainer { pub fn new + Display + Clone>( id: T, base: T, - mut config: Config, + config: Config, logger: &Logger, ) -> Result { let base = base.into(); @@ -1475,26 +1475,18 @@ impl LinuxContainer { .context(format!("Cannot change owner of container {} root", id))?; let spec = config.spec.as_ref().unwrap(); - let linux = spec.linux.as_ref().unwrap(); - - // determine which cgroup driver to take and then assign to config.use_systemd_cgroup - // systemd: "[slice]:[prefix]:[name]" - // fs: "/path_a/path_b" - let cpath = if SYSTEMD_CGROUP_PATH_FORMAT.is_match(linux.cgroups_path.as_str()) { - config.use_systemd_cgroup = true; + let cpath = if config.use_systemd_cgroup { if linux.cgroups_path.len() == 2 { format!("system.slice:kata_agent:{}", id.as_str()) } else { linux.cgroups_path.clone() } + } else if linux.cgroups_path.is_empty() { + format!("/{}", id.as_str()) } else { - config.use_systemd_cgroup = false; - if linux.cgroups_path.is_empty() { - format!("/{}", id.as_str()) - } else { - linux.cgroups_path.clone() - } + // if we have a systemd cgroup path we need to convert it to a fs cgroup path + linux.cgroups_path.replace(':', "/") }; let cgroup_manager: Box = if config.use_systemd_cgroup { diff --git a/src/agent/src/main.rs b/src/agent/src/main.rs index 2c1f41defe..d8e9fc828b 100644 --- a/src/agent/src/main.rs +++ b/src/agent/src/main.rs @@ -339,7 +339,7 @@ async fn start_sandbox( sandbox.lock().await.sender = Some(tx); // vsock:///dev/vsock, port - let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str())?; + let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode)?; server.start().await?; rx.await?; diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index c52d866d65..3be8fed5c8 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -36,7 +36,7 @@ use protocols::health::{ use protocols::types::Interface; use protocols::{agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc}; use rustjail::cgroups::notifier; -use rustjail::container::{BaseContainer, Container, LinuxContainer}; +use rustjail::container::{BaseContainer, Container, LinuxContainer, SYSTEMD_CGROUP_PATH_FORMAT}; use rustjail::process::Process; use rustjail::specconv::CreateOpts; @@ -137,6 +137,7 @@ macro_rules! is_allowed { #[derive(Clone, Debug)] pub struct AgentService { sandbox: Arc>, + init_mode: bool, } impl AgentService { @@ -210,9 +211,20 @@ impl AgentService { // restore the cwd for kata-agent process. defer!(unistd::chdir(&olddir).unwrap()); + // determine which cgroup driver to take and then assign to use_systemd_cgroup + // systemd: "[slice]:[prefix]:[name]" + // fs: "/path_a/path_b" + // If agent is init we can't use systemd cgroup mode, no matter what the host tells us + let cgroups_path = oci.linux.as_ref().map_or("", |linux| &linux.cgroups_path); + let use_systemd_cgroup = if self.init_mode { + false + } else { + SYSTEMD_CGROUP_PATH_FORMAT.is_match(cgroups_path) + }; + let opts = CreateOpts { cgroup_name: "".to_string(), - use_systemd_cgroup: false, + use_systemd_cgroup, no_pivot_root: s.no_pivot_root, no_new_keyring: false, spec: Some(oci.clone()), @@ -1673,9 +1685,11 @@ async fn read_stream(reader: Arc>>, l: usize) -> Resu Ok(content) } -pub fn start(s: Arc>, server_address: &str) -> Result { - let agent_service = - Box::new(AgentService { sandbox: s }) as Box; +pub fn start(s: Arc>, server_address: &str, init_mode: bool) -> Result { + let agent_service = Box::new(AgentService { + sandbox: s, + init_mode, + }) as Box; let agent_worker = Arc::new(agent_service); @@ -2151,6 +2165,7 @@ mod tests { let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), + init_mode: true, }); let req = protocols::agent::UpdateInterfaceRequest::default(); @@ -2168,6 +2183,7 @@ mod tests { let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), + init_mode: true, }); let req = protocols::agent::UpdateRoutesRequest::default(); @@ -2185,6 +2201,7 @@ mod tests { let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), + init_mode: true, }); let req = protocols::agent::AddARPNeighborsRequest::default(); @@ -2318,6 +2335,7 @@ mod tests { let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), + init_mode: true, }); let result = agent_service @@ -2798,6 +2816,7 @@ OtherField:other let sandbox = Sandbox::new(&logger).unwrap(); let agent_service = Box::new(AgentService { sandbox: Arc::new(Mutex::new(sandbox)), + init_mode: true, }); let ctx = mk_ttrpc_context(); diff --git a/src/runtime-rs/arch/powerpc64le-options.mk b/src/runtime-rs/arch/powerpc64le-options.mk new file mode 100644 index 0000000000..0a974680e0 --- /dev/null +++ b/src/runtime-rs/arch/powerpc64le-options.mk @@ -0,0 +1,15 @@ +# Copyright (c) 2019-2022 Alibaba Cloud +# Copyright (c) 2019-2022 Ant Group +# +# SPDX-License-Identifier: Apache-2.0 +# + +MACHINETYPE := pseries +KERNELPARAMS := +MACHINEACCELERATORS := "cap-cfpc=broken,cap-sbbc=broken,cap-ibs=broken,cap-large-decr=off,cap-ccf-assist=off" +CPUFEATURES := pmu=off + +QEMUCMD := qemu-system-ppc64 + +# dragonball binary name +DBCMD := dragonball diff --git a/src/runtime/virtcontainers/mount.go b/src/runtime/virtcontainers/mount.go index 243c13f330..6c2e204208 100644 --- a/src/runtime/virtcontainers/mount.go +++ b/src/runtime/virtcontainers/mount.go @@ -44,6 +44,7 @@ func mountLogger() *logrus.Entry { } func isSystemMount(m string) bool { + m = filepath.Clean(m) for _, p := range systemMountPrefixes { if m == p || strings.HasPrefix(m, p+"/") { return true @@ -54,6 +55,7 @@ func isSystemMount(m string) bool { } func isHostDevice(m string) bool { + m = filepath.Clean(m) if m == "/dev" { return true } diff --git a/src/runtime/virtcontainers/mount_linux_test.go b/src/runtime/virtcontainers/mount_linux_test.go index a34f7c28f3..e5019b401b 100644 --- a/src/runtime/virtcontainers/mount_linux_test.go +++ b/src/runtime/virtcontainers/mount_linux_test.go @@ -249,6 +249,9 @@ func TestIsHostDevice(t *testing.T) { {"/dev/zero", true}, {"/dev/block", true}, {"/mnt/dev/block", false}, + {"/../dev", true}, + {"/../dev/block", true}, + {"/../mnt/dev/block", false}, } for _, test := range tests { diff --git a/src/runtime/virtcontainers/mount_test.go b/src/runtime/virtcontainers/mount_test.go index 6d91d22a7b..c21d00a195 100644 --- a/src/runtime/virtcontainers/mount_test.go +++ b/src/runtime/virtcontainers/mount_test.go @@ -41,6 +41,10 @@ func TestIsSystemMount(t *testing.T) { {"/home", false}, {"/dev/block/", false}, {"/mnt/dev/foo", false}, + {"/../sys", true}, + {"/../sys/", true}, + {"/../sys/fs/cgroup", true}, + {"/../sysfoo", false}, } for _, test := range tests { diff --git a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in index de3c31ed93..937ea89f43 100644 --- a/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in +++ b/tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in @@ -6,6 +6,8 @@ ARG IMAGE_REGISTRY=docker.io FROM ${IMAGE_REGISTRY}/ubuntu:@OS_VERSION@ @SET_PROXY@ +# makedev tries to mknod from postinst +RUN [ -x /usr/bin/systemd-detect-virt ] || ( echo "echo docker" >/usr/bin/systemd-detect-virt && chmod +x /usr/bin/systemd-detect-virt ) RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive \ apt-get --no-install-recommends -y install \ @@ -20,6 +22,7 @@ RUN apt-get update && \ echo "gcc-$gcc_arch-linux-gnu libc6-dev-$libc_arch-cross")) \ git \ make \ + makedev \ multistrap \ musl-tools \ protobuf-compiler diff --git a/tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh b/tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh index daa158bf70..b53749edc5 100644 --- a/tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh +++ b/tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh @@ -34,4 +34,10 @@ EOF # Reduce image size and memory footprint by removing unnecessary files and directories. rm -rf $rootfs_dir/usr/share/{bash-completion,bug,doc,info,lintian,locale,man,menu,misc,pixmaps,terminfo,zsh} + + # Minimal set of device nodes needed when AGENT_INIT=yes so that the + # kernel can properly setup stdout/stdin/stderr for us + pushd $rootfs_dir/dev + MAKEDEV -v console tty ttyS null zero fd + popd } diff --git a/tools/packaging/static-build/firecracker/build-static-firecracker.sh b/tools/packaging/static-build/firecracker/build-static-firecracker.sh index 2d98f4df70..1595754105 100755 --- a/tools/packaging/static-build/firecracker/build-static-firecracker.sh +++ b/tools/packaging/static-build/firecracker/build-static-firecracker.sh @@ -18,6 +18,8 @@ firecracker_repo="${firecracker_repo:-}" firecracker_dir="firecracker" firecracker_version="${firecracker_version:-}" +arch=$(uname -m) + if [ -z "$firecracker_repo" ]; then info "Get firecracker information from runtime versions.yaml" firecracker_url=$(get_from_kata_deps "assets.hypervisor.firecracker.url") @@ -37,5 +39,5 @@ git fetch git checkout ${firecracker_version} sudo ./tools/devtool --unattended build --release -ln -sf ./build/cargo_target/x86_64-unknown-linux-musl/release/firecracker ./firecracker-static -ln -sf ./build/cargo_target/x86_64-unknown-linux-musl/release/jailer ./jailer-static +ln -sf ./build/cargo_target/${arch}-unknown-linux-musl/release/firecracker ./firecracker-static +ln -sf ./build/cargo_target/${arch}-unknown-linux-musl/release/jailer ./jailer-static diff --git a/tools/packaging/static-build/shim-v2/install_go_rust.sh b/tools/packaging/static-build/shim-v2/install_go_rust.sh index db192f673b..078f49d37d 100755 --- a/tools/packaging/static-build/shim-v2/install_go_rust.sh +++ b/tools/packaging/static-build/shim-v2/install_go_rust.sh @@ -50,12 +50,41 @@ EOF trap finish EXIT +go_version=${1:-} rust_version=${2:-} + ARCH=${ARCH:-$(uname -m)} -LIBC=${LIBC:-musl} +case "${ARCH}" in + aarch64) + goarch=arm64 + LIBC=musl + # This is a hack needed as part of Ubuntu 20.04 + if [ ! -f /usr/bin/aarch64-linux-musl-gcc ]; then + ln -sf /usr/bin/musl-gcc /usr/bin/aarch64-linux-musl-gcc + fi + ;; + ppc64le) + goarch=${ARCH} + ARCH=powerpc64le + LIBC=gnu + ;; + s390x) + goarch=${ARCH} + LIBC=gnu + ;; + x86_64) + goarch=amd64 + LIBC=musl + ;; + *) + echo "unsupported architecture $(uname -m)" + exit 1 + ;; +esac + curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSLf | sh -s -- -y --default-toolchain ${rust_version} -t ${ARCH}-unknown-linux-${LIBC} source /root/.cargo/env -rustup target add x86_64-unknown-linux-musl +rustup target add ${ARCH}-unknown-linux-${LIBC} pushd "${tmp_dir}" @@ -70,9 +99,6 @@ done shift $(( $OPTIND - 1 )) - -go_version=${1:-} - if [ -z "$go_version" ];then echo "Missing go" usage 1 @@ -90,14 +116,6 @@ if command -v go; then fi fi -case "$(uname -m)" in - aarch64) goarch="arm64";; - ppc64le) goarch="ppc64le";; - x86_64) goarch="amd64";; - s390x) goarch="s390x";; - *) echo "unsupported architecture: $(uname -m)"; exit 1;; -esac - info "Download go version ${go_version}" kernel_name=$(uname -s) curl -OL "https://storage.googleapis.com/golang/go${go_version}.${kernel_name,,}-${goarch}.tar.gz"