Merge pull request #6517 from gkurz/3.1.0-branch-bump

# Kata Containers 3.1.0

.github/workflows/kata-deploy-push.yaml

@@ -29,12 +29,6 @@ jobs:
           - nydus
     steps:
       - uses: actions/checkout@v2
-      - name: Install docker
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
-
       - name: Build ${{ matrix.asset }}
         if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
         run: |

.github/workflows/kata-deploy-test.yaml

@@ -72,11 +72,6 @@ jobs:
         with:
           ref: ${{ steps.get-PR-ref.outputs.pr-ref }}
 
-      - name: Install docker
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
-
       - name: Build ${{ matrix.asset }}
         run: |
           make "${KATA_ASSET}-tarball"

.github/workflows/release.yaml

@@ -22,11 +22,6 @@ jobs:
           - virtiofsd
     steps:
       - uses: actions/checkout@v2
-      - name: Install docker
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
-
       - name: Build ${{ matrix.asset }}
         run: |
           ./tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh

VERSION

@@ -1 +1 @@
-3.1.0-rc0
+3.1.0

src/agent/Cargo.lock

@@ -276,9 +276,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cgroups-rs"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1"
+checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965"
 dependencies = [
  "libc",
  "log",

src/agent/Cargo.toml

@@ -51,7 +51,7 @@ log = "0.4.11"
 prometheus = { version = "0.13.0", features = ["process"] }
 procfs = "0.12.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.3.1" }
+cgroups = { package = "cgroups-rs", version = "0.3.2" }
 
 # Tracing
 tracing = "0.1.26"

src/agent/rustjail/Cargo.toml

@@ -25,7 +25,7 @@ scan_fmt = "0.2.6"
 regex = "1.5.6"
 path-absolutize = "1.2.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.3.1" }
+cgroups = { package = "cgroups-rs", version = "0.3.2" }
 rlimit = "0.5.3"
 cfg-if = "0.1.0"
 

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -267,6 +267,10 @@ impl CgroupManager for Manager {
     fn as_any(&self) -> Result<&dyn Any> {
         Ok(self)
     }
+
+    fn name(&self) -> &str {
+        "cgroupfs"
+    }
 }
 
 fn set_network_resources(

src/agent/rustjail/src/cgroups/mock.rs

@@ -66,6 +66,10 @@ impl CgroupManager for Manager {
     fn as_any(&self) -> Result<&dyn Any> {
         Ok(self)
     }
+
+    fn name(&self) -> &str {
+        "mock"
+    }
 }
 
 impl Manager {

src/agent/rustjail/src/cgroups/mod.rs

@@ -52,10 +52,12 @@ pub trait Manager {
     fn as_any(&self) -> Result<&dyn Any> {
         Err(anyhow!("not supported!"))
     }
+
+    fn name(&self) -> &str;
 }
 
 impl Debug for dyn Manager + Send + Sync {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
-        write!(f, "CgroupManager")
+        write!(f, "{}", self.name())
     }
 }

src/agent/rustjail/src/cgroups/systemd/manager.rs

@@ -101,6 +101,10 @@ impl CgroupManager for Manager {
     fn as_any(&self) -> Result<&dyn Any> {
         Ok(self)
     }
+
+    fn name(&self) -> &str {
+        "systemd"
+    }
 }
 
 impl Manager {

src/agent/rustjail/src/container.rs

@@ -1449,7 +1449,7 @@ impl LinuxContainer {
     pub fn new<T: Into<String> + Display + Clone>(
         id: T,
         base: T,
-        mut config: Config,
+        config: Config,
         logger: &Logger,
     ) -> Result<Self> {
         let base = base.into();
@@ -1475,26 +1475,18 @@ impl LinuxContainer {
         .context(format!("Cannot change owner of container {} root", id))?;
 
         let spec = config.spec.as_ref().unwrap();
-
         let linux = spec.linux.as_ref().unwrap();
-
-        // determine which cgroup driver to take and then assign to config.use_systemd_cgroup
-        // systemd: "[slice]:[prefix]:[name]"
-        // fs: "/path_a/path_b"
-        let cpath = if SYSTEMD_CGROUP_PATH_FORMAT.is_match(linux.cgroups_path.as_str()) {
-            config.use_systemd_cgroup = true;
+        let cpath = if config.use_systemd_cgroup {
             if linux.cgroups_path.len() == 2 {
                 format!("system.slice:kata_agent:{}", id.as_str())
             } else {
                 linux.cgroups_path.clone()
             }
+        } else if linux.cgroups_path.is_empty() {
+            format!("/{}", id.as_str())
         } else {
-            config.use_systemd_cgroup = false;
-            if linux.cgroups_path.is_empty() {
-                format!("/{}", id.as_str())
-            } else {
-                linux.cgroups_path.clone()
-            }
+            // if we have a systemd cgroup path we need to convert it to a fs cgroup path
+            linux.cgroups_path.replace(':', "/")
         };
 
         let cgroup_manager: Box<dyn Manager + Send + Sync> = if config.use_systemd_cgroup {

src/agent/src/main.rs

@@ -339,7 +339,7 @@ async fn start_sandbox(
     sandbox.lock().await.sender = Some(tx);
 
     // vsock:///dev/vsock, port
-    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str())?;
+    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str(), init_mode)?;
     server.start().await?;
 
     rx.await?;

src/agent/src/rpc.rs

@@ -36,7 +36,7 @@ use protocols::health::{
 use protocols::types::Interface;
 use protocols::{agent_ttrpc_async as agent_ttrpc, health_ttrpc_async as health_ttrpc};
 use rustjail::cgroups::notifier;
-use rustjail::container::{BaseContainer, Container, LinuxContainer};
+use rustjail::container::{BaseContainer, Container, LinuxContainer, SYSTEMD_CGROUP_PATH_FORMAT};
 use rustjail::process::Process;
 use rustjail::specconv::CreateOpts;
 
@@ -137,6 +137,7 @@ macro_rules! is_allowed {
 #[derive(Clone, Debug)]
 pub struct AgentService {
     sandbox: Arc<Mutex<Sandbox>>,
+    init_mode: bool,
 }
 
 impl AgentService {
@@ -210,9 +211,20 @@ impl AgentService {
         // restore the cwd for kata-agent process.
         defer!(unistd::chdir(&olddir).unwrap());
 
+        // determine which cgroup driver to take and then assign to use_systemd_cgroup
+        // systemd: "[slice]:[prefix]:[name]"
+        // fs: "/path_a/path_b"
+        // If agent is init we can't use systemd cgroup mode, no matter what the host tells us
+        let cgroups_path = oci.linux.as_ref().map_or("", |linux| &linux.cgroups_path);
+        let use_systemd_cgroup = if self.init_mode {
+            false
+        } else {
+            SYSTEMD_CGROUP_PATH_FORMAT.is_match(cgroups_path)
+        };
+
         let opts = CreateOpts {
             cgroup_name: "".to_string(),
-            use_systemd_cgroup: false,
+            use_systemd_cgroup,
             no_pivot_root: s.no_pivot_root,
             no_new_keyring: false,
             spec: Some(oci.clone()),
@@ -1673,9 +1685,11 @@ async fn read_stream(reader: Arc<Mutex<ReadHalf<PipeStream>>>, l: usize) -> Resu
     Ok(content)
 }
 
-pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str) -> Result<TtrpcServer> {
-    let agent_service =
-        Box::new(AgentService { sandbox: s }) as Box<dyn agent_ttrpc::AgentService + Send + Sync>;
+pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str, init_mode: bool) -> Result<TtrpcServer> {
+    let agent_service = Box::new(AgentService {
+        sandbox: s,
+        init_mode,
+    }) as Box<dyn agent_ttrpc::AgentService + Send + Sync>;
 
     let agent_worker = Arc::new(agent_service);
 
@@ -2151,6 +2165,7 @@ mod tests {
 
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
+            init_mode: true,
         });
 
         let req = protocols::agent::UpdateInterfaceRequest::default();
@@ -2168,6 +2183,7 @@ mod tests {
 
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
+            init_mode: true,
         });
 
         let req = protocols::agent::UpdateRoutesRequest::default();
@@ -2185,6 +2201,7 @@ mod tests {
 
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
+            init_mode: true,
         });
 
         let req = protocols::agent::AddARPNeighborsRequest::default();
@@ -2318,6 +2335,7 @@ mod tests {
 
             let agent_service = Box::new(AgentService {
                 sandbox: Arc::new(Mutex::new(sandbox)),
+                init_mode: true,
             });
 
             let result = agent_service
@@ -2798,6 +2816,7 @@ OtherField:other
         let sandbox = Sandbox::new(&logger).unwrap();
         let agent_service = Box::new(AgentService {
             sandbox: Arc::new(Mutex::new(sandbox)),
+            init_mode: true,
         });
 
         let ctx = mk_ttrpc_context();

src/libs/Cargo.lock

@@ -110,9 +110,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cgroups-rs"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1"
+checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965"
 dependencies = [
  "libc",
  "log",

src/libs/kata-sys-util/Cargo.toml

@@ -12,7 +12,7 @@ edition = "2018"
 
 [dependencies]
 byteorder = "1.4.3"
-cgroups = { package = "cgroups-rs", version = "0.3.1" }
+cgroups = { package = "cgroups-rs", version = "0.3.2" }
 chrono = "0.4.0"
 common-path = "=1.0.0"
 fail = "0.5.0"

src/runtime-rs/Cargo.lock

@@ -401,9 +401,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cgroups-rs"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1"
+checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965"
 dependencies = [
  "libc",
  "log",

src/runtime-rs/arch/powerpc64le-options.mk

@@ -0,0 +1,15 @@
+# Copyright (c) 2019-2022 Alibaba Cloud
+# Copyright (c) 2019-2022 Ant Group
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+MACHINETYPE := pseries
+KERNELPARAMS :=
+MACHINEACCELERATORS := "cap-cfpc=broken,cap-sbbc=broken,cap-ibs=broken,cap-large-decr=off,cap-ccf-assist=off"
+CPUFEATURES := pmu=off
+
+QEMUCMD := qemu-system-ppc64
+
+# dragonball binary name
+DBCMD := dragonball

src/runtime-rs/crates/resource/Cargo.toml

@@ -14,7 +14,7 @@ anyhow = "^1.0"
 async-trait = "0.1.48"
 bitflags = "1.2.1"
 byte-unit = "4.0.14"
-cgroups-rs = "0.3.1"
+cgroups-rs = "0.3.2"
 futures = "0.3.11"
 lazy_static = "1.4.0"
 libc = ">=0.2.39"

src/runtime/virtcontainers/mount.go

@@ -44,6 +44,7 @@ func mountLogger() *logrus.Entry {
 }
 
 func isSystemMount(m string) bool {
+	m = filepath.Clean(m)
 	for _, p := range systemMountPrefixes {
 		if m == p || strings.HasPrefix(m, p+"/") {
 			return true
@@ -54,6 +55,7 @@ func isSystemMount(m string) bool {
 }
 
 func isHostDevice(m string) bool {
+	m = filepath.Clean(m)
 	if m == "/dev" {
 		return true
 	}

src/runtime/virtcontainers/mount_linux_test.go

@@ -249,6 +249,9 @@ func TestIsHostDevice(t *testing.T) {
 		{"/dev/zero", true},
 		{"/dev/block", true},
 		{"/mnt/dev/block", false},
+		{"/../dev", true},
+		{"/../dev/block", true},
+		{"/../mnt/dev/block", false},
 	}
 
 	for _, test := range tests {

src/runtime/virtcontainers/mount_test.go

@@ -41,6 +41,10 @@ func TestIsSystemMount(t *testing.T) {
 		{"/home", false},
 		{"/dev/block/", false},
 		{"/mnt/dev/foo", false},
+		{"/../sys", true},
+		{"/../sys/", true},
+		{"/../sys/fs/cgroup", true},
+		{"/../sysfoo", false},
 	}
 
 	for _, test := range tests {

src/tools/agent-ctl/Cargo.lock

@@ -250,9 +250,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cgroups-rs"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1"
+checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965"
 dependencies = [
  "libc",
  "log",

src/tools/runk/Cargo.lock

@@ -233,9 +233,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cgroups-rs"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d5761f3a351b92e0e02a31ca418190bb323edb0d4fce0109b6dba673dc3fdc1"
+checksum = "5b098e7c3a70d03c288fa0a96ccf13e770eb3d78c4cc0e1549b3c13215d5f965"
 dependencies = [
  "libc",
  "log",

src/tools/runk/libcontainer/Cargo.toml

@@ -20,7 +20,7 @@ chrono = { version = "0.4.19", features = ["serde"] }
 serde = { version = "1.0.133", features = ["derive"] }
 serde_json = "1.0.74"
 scopeguard = "1.1.0"
-cgroups = { package = "cgroups-rs", version = "0.3.1" }
+cgroups = { package = "cgroups-rs", version = "0.3.2" }
 procfs = "0.14.0"
 
 [dev-dependencies]

tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in

@@ -6,6 +6,8 @@ ARG IMAGE_REGISTRY=docker.io
 FROM ${IMAGE_REGISTRY}/ubuntu:@OS_VERSION@
 @SET_PROXY@
 
+# makedev tries to mknod from postinst
+RUN [ -x /usr/bin/systemd-detect-virt ] || ( echo "echo docker" >/usr/bin/systemd-detect-virt && chmod +x /usr/bin/systemd-detect-virt )
 RUN apt-get update && \
     DEBIAN_FRONTEND=noninteractive \
     apt-get --no-install-recommends -y install \
@@ -20,6 +22,7 @@ RUN apt-get update && \
          echo "gcc-$gcc_arch-linux-gnu libc6-dev-$libc_arch-cross")) \
     git \
     make \
+    makedev \
     multistrap \
     musl-tools \
     protobuf-compiler

tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh

@@ -34,4 +34,10 @@ EOF
 
 	# Reduce image size and memory footprint by removing unnecessary files and directories.
 	rm -rf $rootfs_dir/usr/share/{bash-completion,bug,doc,info,lintian,locale,man,menu,misc,pixmaps,terminfo,zsh}
+
+	# Minimal set of device nodes needed when AGENT_INIT=yes so that the
+	# kernel can properly setup stdout/stdin/stderr for us
+	pushd $rootfs_dir/dev
+	MAKEDEV -v console tty ttyS null zero fd
+	popd
 }

tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml

@@ -18,7 +18,7 @@ spec:
           katacontainers.io/kata-runtime: cleanup
       containers:
       - name: kube-kata-cleanup
-        image: quay.io/kata-containers/kata-deploy:latest
+        image: quay.io/kata-containers/kata-deploy:3.1.0
         imagePullPolicy: Always
         command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh reset" ]
         env:

tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml

@@ -16,7 +16,7 @@ spec:
       serviceAccountName: kata-label-node
       containers:
       - name: kube-kata
-        image: quay.io/kata-containers/kata-deploy:latest
+        image: quay.io/kata-containers/kata-deploy:3.1.0
         imagePullPolicy: Always
         lifecycle:
           preStop:

tools/packaging/static-build/firecracker/build-static-firecracker.sh

@@ -18,6 +18,8 @@ firecracker_repo="${firecracker_repo:-}"
 firecracker_dir="firecracker"
 firecracker_version="${firecracker_version:-}"
 
+arch=$(uname -m)
+
 if [ -z "$firecracker_repo" ]; then
 	info "Get firecracker information from runtime versions.yaml"
 	firecracker_url=$(get_from_kata_deps "assets.hypervisor.firecracker.url")
@@ -37,5 +39,5 @@ git fetch
 git checkout ${firecracker_version}
 sudo ./tools/devtool --unattended build --release
 
-ln -sf ./build/cargo_target/x86_64-unknown-linux-musl/release/firecracker ./firecracker-static
-ln -sf ./build/cargo_target/x86_64-unknown-linux-musl/release/jailer ./jailer-static
+ln -sf ./build/cargo_target/${arch}-unknown-linux-musl/release/firecracker ./firecracker-static
+ln -sf ./build/cargo_target/${arch}-unknown-linux-musl/release/jailer ./jailer-static

tools/packaging/static-build/shim-v2/install_go_rust.sh

@@ -50,12 +50,41 @@ EOF
 
 trap finish EXIT
 
+go_version=${1:-}
 rust_version=${2:-}
+
 ARCH=${ARCH:-$(uname -m)}
-LIBC=${LIBC:-musl}
+case "${ARCH}" in
+	aarch64)
+		goarch=arm64
+		LIBC=musl
+		# This is a hack needed as part of Ubuntu 20.04
+		if [ ! -f /usr/bin/aarch64-linux-musl-gcc ]; then
+			ln -sf /usr/bin/musl-gcc /usr/bin/aarch64-linux-musl-gcc
+		fi
+		;;
+	ppc64le) 
+		goarch=${ARCH}
+		ARCH=powerpc64le
+		LIBC=gnu
+		;;
+	s390x)
+		goarch=${ARCH}
+		LIBC=gnu
+		;;
+	x86_64)
+		goarch=amd64
+		LIBC=musl
+		;;
+	*)
+		echo "unsupported architecture $(uname -m)"
+		exit 1
+		;;
+esac
+
 curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSLf | sh -s -- -y --default-toolchain ${rust_version} -t ${ARCH}-unknown-linux-${LIBC}
 source /root/.cargo/env
-rustup target add x86_64-unknown-linux-musl
+rustup target add ${ARCH}-unknown-linux-${LIBC}
 
 pushd "${tmp_dir}"
 
@@ -70,9 +99,6 @@ done
 
 shift $(( $OPTIND - 1 ))
 
-
-go_version=${1:-}
-
 if [ -z "$go_version" ];then
 	echo "Missing go"
 	usage 1
@@ -90,14 +116,6 @@ if command -v go; then
 	fi
 fi
 
-case "$(uname -m)" in
-	aarch64) goarch="arm64";;
-	ppc64le) goarch="ppc64le";;
-	x86_64) goarch="amd64";;
-	s390x) goarch="s390x";;
-	*) echo "unsupported architecture: $(uname -m)"; exit 1;;
-esac
-
 info "Download go version ${go_version}"
 kernel_name=$(uname -s)
 curl -OL "https://storage.googleapis.com/golang/go${go_version}.${kernel_name,,}-${goarch}.tar.gz"