Merge pull request #4024 from bergwolf/2.4.0-branch-bump

# Kata Containers 2.4.0

.github/workflows/add-pr-sizing-label.yaml

@@ -1,38 +0,0 @@
-# Copyright (c) 2022 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-name: Add PR sizing label
-
-on:
-  pull_request_target:
-    types:
-      - opened
-      - reopened
-      - synchronize
-
-jobs:
-  add-pr-size-label:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v1
-
-      - name: Install PR sizing label script
-        run: |
-          # Clone into a temporary directory to avoid overwriting
-          # any existing github directory.
-          pushd $(mktemp -d) &>/dev/null
-          git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
-          sudo install pr-add-size-label.sh /usr/local/bin
-          popd &>/dev/null
-
-      - name: Add PR sizing label
-        env:
-          GITHUB_TOKEN: ${{ secrets.KATA_GITHUB_ACTIONS_PR_SIZE_TOKEN }}
-        run: |
-          pr=${{ github.event.number }}
-          sudo apt -y install diffstat patchutils
-
-          pr-add-size-label.sh -p "$pr"

.github/workflows/commit-message-check.yaml

@@ -10,7 +10,7 @@ env:
   error_msg: |+
     See the document below for help on formatting commits for the project.
 
-    https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md#patch-format
+    https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md#patch-format
 
 jobs:
   commit-message-check:

.github/workflows/docs-url-alive-check.yaml

@@ -1,44 +0,0 @@
-on:
-  schedule:
-    - cron:  '0 23 * * 0'
-
-name: Docs URL Alive Check
-jobs:
-  test:
-    strategy:
-      matrix:
-        go-version: [1.17.x]
-        os: [ubuntu-20.04]
-    runs-on: ${{ matrix.os }}
-    env:
-      target_branch: ${{ github.base_ref }}
-    steps:
-    - name: Install Go
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ matrix.go-version }}
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Set env
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Checkout code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
-    - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    # docs url alive check
-    - name: Docs URL Alive Check
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make docs-url-alive-check

.github/workflows/release.yaml

@@ -26,7 +26,6 @@ jobs:
 
       - name: Build ${{ matrix.asset }}
         run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh
           ./tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh --build="${KATA_ASSET}"
           build_dir=$(readlink -f build)
           # store-artifact does not work with symlink

.gitignore

@@ -9,5 +9,4 @@ src/agent/src/version.rs
 src/agent/kata-agent.service
 src/agent/protocols/src/*.rs
 !src/agent/protocols/src/lib.rs
-build
 

Makefile

@@ -39,16 +39,10 @@ generate-protocols:
 static-checks: build
 	bash ci/static-checks.sh
 
-docs-url-alive-check:
-	bash ci/docs-url-alive-check.sh
-
 .PHONY: \
 	all \
 	binary-tarball \
 	default \
 	install-binary-tarball \
 	logging-crate-tests \
-	static-checks \
-	docs-url-alive-check
-
-
+	static-checks

VERSION

@@ -1 +1 @@
-2.5.0-alpha0
+2.4.0

ci/docs-url-alive-check.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-#
-# Copyright (c) 2021 Easystack Inc.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-set -e
-
-cidir=$(dirname "$0")
-source "${cidir}/lib.sh"
-
-run_docs_url_alive_check

ci/install_libseccomp.sh

@@ -19,7 +19,7 @@ source "${tests_repo_dir}/.ci/lib.sh"
 # fail. So let's ensure they are unset here.
 unset PREFIX DESTDIR
 
-arch=${ARCH:-$(uname -m)}
+arch=$(uname -m)
 workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
 
 # Variables for libseccomp
@@ -70,8 +70,7 @@ build_and_install_gperf() {
     curl -sLO "${gperf_tarball_url}"
     tar -xf "${gperf_tarball}"
     pushd "gperf-${gperf_version}"
-    # Unset $CC for configure, we will always use native for gperf
-    CC= ./configure --prefix="${gperf_install_dir}"
+    ./configure --prefix="${gperf_install_dir}"
     make
     make install
     export PATH=$PATH:"${gperf_install_dir}"/bin
@@ -85,7 +84,7 @@ build_and_install_libseccomp() {
     curl -sLO "${libseccomp_tarball_url}"
     tar -xf "${libseccomp_tarball}"
     pushd "libseccomp-${libseccomp_version}"
-    ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static --host="${arch}"
+    ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static
     make
     make install
     popd

ci/install_musl.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Copyright (c) 2020 Ant Group
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -e
+
+install_aarch64_musl() {
+	local arch=$(uname -m)
+	if [ "${arch}" == "aarch64" ]; then
+		local musl_tar="${arch}-linux-musl-native.tgz"
+		local musl_dir="${arch}-linux-musl-native"
+		pushd /tmp
+		if curl -sLO --fail https://musl.cc/${musl_tar}; then
+			tar -zxf ${musl_tar}
+			mkdir -p /usr/local/musl/
+			cp -r ${musl_dir}/* /usr/local/musl/
+		fi
+		popd
+	fi
+}
+
+install_aarch64_musl

ci/lib.sh

@@ -44,12 +44,3 @@ run_go_test()
 	clone_tests_repo
 	bash "$tests_repo_dir/.ci/go-test.sh"
 }
-
-run_docs_url_alive_check()
-{
-	clone_tests_repo
-	# Make sure we have the targeting branch
-	git remote set-branches --add origin "${branch}"
-	git fetch -a
-	bash "$tests_repo_dir/.ci/static-checks.sh" --docs --all "github.com/kata-containers/kata-containers"
-}

docs/README.md

@@ -30,6 +30,7 @@ See the [how-to documentation](how-to).
 * [GPU Passthrough with Kata](./use-cases/GPU-passthrough-and-Kata.md)
 * [SR-IOV with Kata](./use-cases/using-SRIOV-and-kata.md)
 * [Intel QAT with Kata](./use-cases/using-Intel-QAT-and-kata.md)
+* [VPP with Kata](./use-cases/using-vpp-and-kata.md)
 * [SPDK vhost-user with Kata](./use-cases/using-SPDK-vhostuser-and-kata.md)
 * [Intel SGX with Kata](./use-cases/using-Intel-SGX-and-kata.md)
 

docs/design/VSocks.md

@@ -67,15 +67,22 @@ Using a proxy for multiplexing the connections between the VM and the host uses
 4.5MB per [POD][2]. In a high density deployment this could add up to GBs of
 memory that could have been used to host more PODs. When we talk about density
 each kilobyte matters and it might be the decisive factor between run another
-POD or not. Before making the decision not to use VSOCKs, you should ask
+POD or not. For example if you have 500 PODs running in a server, the same
+amount of [`kata-proxy`][3] processes will be running and consuming for around
+2250MB of RAM. Before making the decision not to use VSOCKs, you should ask
 yourself, how many more containers can run with the memory RAM consumed by the
 Kata proxies?
 
 ### Reliability
 
+[`kata-proxy`][3] is in charge of multiplexing the connections between virtual
+machine and host processes, if it dies all connections get broken. For example
+if you have a [POD][2] with 10 containers running, if `kata-proxy` dies it would
+be impossible to contact your containers, though they would still be running.
 Since communication via VSOCKs is direct, the only way to lose communication
 with the containers is if the VM itself or the `containerd-shim-kata-v2` dies, if this happens
 the containers are removed automatically.
 
 [1]: https://wiki.qemu.org/Features/VirtioVsock
 [2]: ./vcpu-handling.md#virtual-cpus-and-kubernetes-pods
+[3]: https://github.com/kata-containers/proxy

docs/design/vcpu-handling.md

@@ -2,15 +2,24 @@
 
 ## Default number of virtual CPUs
 
-Before starting a container, the [runtime][4] reads the `default_vcpus` option
-from the [configuration file][5] to determine the number of virtual CPUs
+Before starting a container, the [runtime][6] reads the `default_vcpus` option
+from the [configuration file][7] to determine the number of virtual CPUs
 (vCPUs) needed to start the virtual machine. By default, `default_vcpus` is
 equal to 1 for fast boot time and a small memory footprint per virtual machine.
 Be aware that increasing this value negatively impacts the virtual machine's
 boot time and memory footprint.
 In general, we recommend that you do not edit this variable, unless you know
 what are you doing. If your container needs more than one vCPU, use
-[Kubernetes `cpu` limits][1] to assign more resources.
+[docker `--cpus`][1], [docker update][4], or [Kubernetes `cpu` limits][2] to
+assign more resources.
+
+*Docker*
+
+```sh
+$ docker run --name foo -ti --cpus 2 debian bash
+$ docker update --cpus 4 foo
+```
+
 
 *Kubernetes*
 
@@ -40,7 +49,7 @@ $ sudo -E kubectl create -f ~/cpu-demo.yaml
 ## Virtual CPUs and Kubernetes pods
 
 A Kubernetes pod is a group of one or more containers, with shared storage and
-network, and a specification for how to run the containers [[specification][2]].
+network, and a specification for how to run the containers [[specification][3]].
 In Kata Containers this group of containers, which is called a sandbox, runs inside
 the same virtual machine. If you do not specify a CPU constraint, the runtime does
 not add more vCPUs and the container is not placed inside a CPU cgroup.
@@ -64,7 +73,13 @@ constraints with each container trying to consume 100% of vCPU, the resources
 divide in two parts, 50% of vCPU for each container because your virtual
 machine does not have enough resources to satisfy containers needs. If you want
 to give access to a greater or lesser portion of vCPUs to a specific container,
-use [Kubernetes `cpu` requests][1].
+use [`docker --cpu-shares`][1] or [Kubernetes `cpu` requests][2].
+
+*Docker*
+
+```sh
+$ docker run -ti --cpus-shares=512 debian bash
+```
 
 *Kubernetes*
 
@@ -94,9 +109,10 @@ $ sudo -E kubectl create -f ~/cpu-demo.yaml
 Before running containers without CPU constraint, consider that your containers
 are not running alone. Since your containers run inside a virtual machine other
 processes use the vCPUs as well (e.g. `systemd` and the Kata Containers
-[agent][3]). In general, we recommend setting `default_vcpus` equal to 1 to
+[agent][5]). In general, we recommend setting `default_vcpus` equal to 1 to
 allow non-container processes to run on this vCPU and to specify a CPU
-constraint for each container.
+constraint for each container. If your container is already running and needs
+more vCPUs, you can add more using [docker update][4].
 
 ## Container with CPU constraint
 
@@ -105,7 +121,7 @@ constraints using the following formula: `vCPUs = ceiling( quota / period )`, wh
 `quota` specifies the number of microseconds per CPU Period that the container is
 guaranteed CPU access and `period` specifies the CPU CFS scheduler period of time
 in microseconds. The result determines the number of vCPU to hot plug into the
-virtual machine. Once the vCPUs have been added, the [agent][3] places the
+virtual machine. Once the vCPUs have been added, the [agent][5] places the
 container inside a CPU cgroup. This placement allows the container to use only
 its assigned resources.
 
@@ -122,6 +138,25 @@ the virtual machine starts with 8 vCPUs and 1 vCPUs is added and assigned
 to the container. Non-container processes might be able to use 8 vCPUs but they
 use a maximum 1 vCPU, hence 7 vCPUs might not be used.
 
+
+*Container without CPU constraint*
+
+```sh
+$ docker run -ti debian bash -c "nproc; cat /sys/fs/cgroup/cpu,cpuacct/cpu.cfs_*"
+1       # number of vCPUs
+100000  # cfs period
+-1      # cfs quota
+```
+
+*Container with CPU constraint*
+
+```sh
+docker run --cpus 4 -ti debian bash -c "nproc; cat /sys/fs/cgroup/cpu,cpuacct/cpu.cfs_*"
+5       # number of vCPUs
+100000  # cfs period
+400000  # cfs quota
+```
+
 ## Virtual CPU handling without hotplug
 
 In some cases, the hardware and/or software architecture being utilized does not support
@@ -148,8 +183,11 @@ the container's `spec` will provide the sizing information directly. If these ar
 calculate the number of CPUs required for the workload and augment this by `default_vcpus`
 configuration option, and use this for the virtual machine size.
 
-[1]: https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource
-[2]: https://kubernetes.io/docs/concepts/workloads/pods/pod/
-[3]: ../../src/agent
-[4]: ../../src/runtime
-[5]: ../../src/runtime/README.md#configuration
+
+[1]: https://docs.docker.com/config/containers/resource_constraints/#cpu
+[2]: https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource
+[3]: https://kubernetes.io/docs/concepts/workloads/pods/pod/
+[4]: https://docs.docker.com/engine/reference/commandline/update/
+[5]: ../../src/agent
+[6]: ../../src/runtime
+[7]: ../../src/runtime/README.md#configuration

docs/how-to/how-to-run-docker-with-kata.md

@@ -48,9 +48,9 @@ Running Docker containers Kata Containers requires care because `VOLUME`s specif
 kataShared on / type virtiofs (rw,relatime,dax)
 ```
 
-`kataShared` mount types are powered by [`virtio-fs`](https://virtio-fs.gitlab.io/), a marked improvement over `virtio-9p`, thanks to [PR #1016](https://github.com/kata-containers/runtime/pull/1016). While `virtio-fs` is normally an excellent choice, in the case of DinD workloads `virtio-fs` causes an issue -- [it *cannot* be used as a "upper layer" of `overlayfs` without a custom patch](http://lists.katacontainers.io/pipermail/kata-dev/2020-January/001216.html).
+`kataShared` mount types are powered by [`virtio-fs`][virtio-fs], a marked improvement over `virtio-9p`, thanks to [PR #1016](https://github.com/kata-containers/runtime/pull/1016). While `virtio-fs` is normally an excellent choice, in the case of DinD workloads `virtio-fs` causes an issue -- [it *cannot* be used as a "upper layer" of `overlayfs` without a custom patch](http://lists.katacontainers.io/pipermail/kata-dev/2020-January/001216.html).
 
-As `/var/lib/docker` is a `VOLUME` specified by DinD (i.e. the `docker` images tagged `*-dind`/`*-dind-rootless`), `docker` will fail to start (or even worse, silently pick a worse storage driver like `vfs`) when started in a Kata Container. Special measures must be taken when running DinD-powered workloads in Kata Containers.
+As `/var/lib/docker` is a `VOLUME` specified by DinD (i.e. the `docker` images tagged `*-dind`/`*-dind-rootless`), `docker` fill fail to start (or even worse, silently pick a worse storage driver like `vfs`) when started in a Kata Container. Special measures must be taken when running DinD-powered workloads in Kata Containers.
 
 ## Workarounds/Solutions
 
@@ -58,7 +58,7 @@ Thanks to various community contributions (see [issue references below](#referen
 
 ### Use a memory backed volume
 
-For small workloads (small container images, without much generated filesystem load), a memory-backed volume is sufficient. Kubernetes supports a variant of  [the `EmptyDir` volume](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir), which allows for memdisk-backed storage -- the the `medium: Memory`. An example of a `Pod` using such a setup [was contributed](https://github.com/kata-containers/runtime/issues/1429#issuecomment-477385283), and is reproduced below:
+For small workloads (small container images, without much generated filesystem load), a memory-backed volume is sufficient. Kubernetes supports a variant of  [the `EmptyDir` volume][k8s-emptydir], which allows for memdisk-backed storage -- the [the `medium: Memory` ][k8s-memory-volume-type]. An example of a `Pod` using such a setup [was contributed](https://github.com/kata-containers/runtime/issues/1429#issuecomment-477385283), and is reproduced below:
 
 ```yaml
 apiVersion: v1

docs/how-to/how-to-use-kata-containers-with-acrn.md

@@ -101,7 +101,7 @@ Start an ACRN based Kata Container,
 $ sudo docker run -ti --runtime=kata-runtime busybox sh
 ```
 
-You will see ACRN(`acrn-dm`) is now running on your system, as well as a `kata-shim`. You should obtain an interactive shell prompt. Verify that all the Kata processes terminate once you exit the container.
+You will see ACRN(`acrn-dm`) is now running on your system, as well as a `kata-shim`, `kata-proxy`. You should obtain an interactive shell prompt. Verify that all the Kata processes terminate once you exit the container.
 
 ```bash
 $ ps -ef | grep -E "kata|acrn"

docs/use-cases/using-vpp-and-kata.md

@@ -0,0 +1,76 @@
+# Setup to run VPP
+
+The Data Plane Development Kit (DPDK) is a set of libraries and drivers for
+fast packet processing. Vector Packet Processing (VPP) is a platform
+extensible framework that provides out-of-the-box production quality
+switch and router functionality. VPP is a high performance packet-processing
+stack that can run on commodity CPUs. Enabling VPP with DPDK support can
+yield significant performance improvements over a Linux\* bridge providing a
+switch with DPDK VHOST-USER ports.
+
+For more information about VPP visit their [wiki](https://wiki.fd.io/view/VPP).
+
+## Install and configure Kata Containers
+
+Follow  the [Kata Containers setup instructions](../Developer-Guide.md).
+
+In order to make use of VHOST-USER based interfaces, the container needs to be backed
+by huge pages. `HugePages` support is required for the large memory pool allocation used for
+DPDK packet buffers.  This is a feature which must be configured within the Linux Kernel. See
+[the DPDK documentation](https://doc.dpdk.org/guides/linux_gsg/sys_reqs.html#use-of-hugepages-in-the-linux-environment)
+for details on how to enable for the host. After enabling huge pages support on the host system,
+update the Kata configuration to enable huge page support in the guest kernel:
+
+```
+$ sudo sed -i -e 's/^# *\(enable_hugepages\).*=.*$/\1 = true/g' /usr/share/defaults/kata-containers/configuration.toml
+```
+
+
+## Install VPP
+
+Follow the [VPP installation instructions](https://wiki.fd.io/view/VPP/Installing_VPP_binaries_from_packages).
+
+After a successful installation, your host system is ready to start
+connecting Kata Containers with VPP bridges.
+
+### Install the VPP Docker\* plugin
+
+To create a Docker network and connect Kata Containers easily to that network through
+Docker, install a VPP Docker plugin.
+
+To install the plugin, follow the [plugin installation instructions](https://github.com/clearcontainers/vpp).
+
+This VPP plugin allows the creation of a VPP network. Every container added
+to this network is connected through an L2 bridge-domain provided by VPP.
+
+## Example: Launch two Kata Containers using VPP
+
+To use VPP, use Docker to create a network that makes use of VPP.
+For example:
+
+```
+$ sudo docker network create -d=vpp --ipam-driver=vpp --subnet=192.168.1.0/24 --gateway=192.168.1.1  vpp_net
+```
+
+Test connectivity by launching two containers:
+```
+$ sudo docker run --runtime=kata-runtime --net=vpp_net --ip=192.168.1.2 --mac-address=CA:FE:CA:FE:01:02 -it busybox bash -c "ip a; ip route; sleep 300"
+
+$ sudo docker run --runtime=kata-runtime --net=vpp_net --ip=192.168.1.3 --mac-address=CA:FE:CA:FE:01:03 -it busybox bash -c "ip a; ip route; ping 192.168.1.2"
+```
+
+These commands setup two Kata Containers connected via a VPP L2 bridge
+domain. The first of the two VMs displays the networking details and then
+sleeps providing a period of time for it to be pinged. The second
+VM displays its networking details and then pings the first VM, verifying
+connectivity between them.
+
+After verifying connectivity, cleanup with the following commands:
+
+```
+$ sudo docker kill $(sudo docker ps --no-trunc -aq)
+$ sudo docker rm $(sudo docker ps --no-trunc -aq)
+$ sudo docker network rm vpp_net
+$ sudo service vpp stop
+```
+

src/agent/rustjail/src/cgroups/notifier.rs

@@ -151,12 +151,12 @@ async fn register_memory_event(
     let eventfd = eventfd(0, EfdFlags::EFD_CLOEXEC)?;
 
     let event_control_path = Path::new(&cg_dir).join("cgroup.event_control");
-
-    let data = if arg.is_empty() {
-        format!("{} {}", eventfd, event_file.as_raw_fd())
+    let data;
+    if arg.is_empty() {
+        data = format!("{} {}", eventfd, event_file.as_raw_fd());
     } else {
-        format!("{} {} {}", eventfd, event_file.as_raw_fd(), arg)
-    };
+        data = format!("{} {} {}", eventfd, event_file.as_raw_fd(), arg);
+    }
 
     fs::write(&event_control_path, data)?;
 

src/agent/rustjail/src/lib.rs

@@ -351,12 +351,13 @@ fn seccomp_grpc_to_oci(sec: &grpc::LinuxSeccomp) -> oci::LinuxSeccomp {
 
         for sys in sec.Syscalls.iter() {
             let mut args = Vec::new();
+            let errno_ret: u32;
 
-            let errno_ret: u32 = if sys.has_errnoret() {
-                sys.get_errnoret()
+            if sys.has_errnoret() {
+                errno_ret = sys.get_errnoret();
             } else {
-                libc::EPERM as u32
-            };
+                errno_ret = libc::EPERM as u32;
+            }
 
             for arg in sys.Args.iter() {
                 args.push(oci::LinuxSeccompArg {

src/agent/rustjail/src/utils.rs

@@ -97,13 +97,12 @@ mod tests {
         let temp_passwd = format!("{}/passwd", tmpdir_path);
 
         let mut tempf = File::create(temp_passwd.as_str()).unwrap();
-        let passwd_entries = "root:x:0:0:root:/root0:/bin/bash
-root:x:1:0:root:/root1:/bin/bash
-#root:x:1:0:root:/rootx:/bin/bash
-root:x:2:0:root:/root2:/bin/bash
-root:x:3:0:root:/root3
-root:x:3:0:root:/root3:/bin/bash";
-        writeln!(tempf, "{}", passwd_entries).unwrap();
+        writeln!(tempf, "root:x:0:0:root:/root0:/bin/bash").unwrap();
+        writeln!(tempf, "root:x:1:0:root:/root1:/bin/bash").unwrap();
+        writeln!(tempf, "#root:x:1:0:root:/rootx:/bin/bash").unwrap();
+        writeln!(tempf, "root:x:2:0:root:/root2:/bin/bash").unwrap();
+        writeln!(tempf, "root:x:3:0:root:/root3").unwrap();
+        writeln!(tempf, "root:x:3:0:root:/root3:/bin/bash").unwrap();
 
         let entry = get_entry_by_uid(0, temp_passwd.as_str()).unwrap();
         assert_eq!(entry.dir.as_str(), "/root0");

src/agent/src/main.rs

@@ -125,7 +125,9 @@ fn announce(logger: &Logger, config: &AgentConfig) {
 // output to the vsock port specified, or stdout.
 async fn create_logger_task(rfd: RawFd, vsock_port: u32, shutdown: Receiver<bool>) -> Result<()> {
     let mut reader = PipeStream::from_fd(rfd);
-    let mut writer: Box<dyn AsyncWrite + Unpin + Send> = if vsock_port > 0 {
+    let mut writer: Box<dyn AsyncWrite + Unpin + Send>;
+
+    if vsock_port > 0 {
         let listenfd = socket::socket(
             AddressFamily::Vsock,
             SockType::Stream,
@@ -137,10 +139,10 @@ async fn create_logger_task(rfd: RawFd, vsock_port: u32, shutdown: Receiver<bool
         socket::bind(listenfd, &addr)?;
         socket::listen(listenfd, 1)?;
 
-        Box::new(util::get_vsock_stream(listenfd).await?)
+        writer = Box::new(util::get_vsock_stream(listenfd).await?);
     } else {
-        Box::new(tokio::io::stdout())
-    };
+        writer = Box::new(tokio::io::stdout());
+    }
 
     let _ = util::interruptable_io_copier(&mut reader, &mut writer, shutdown).await;
 

src/agent/src/metrics.rs

@@ -344,25 +344,25 @@ fn set_gauge_vec_meminfo(gv: &prometheus::GaugeVec, meminfo: &procfs::Meminfo) {
 #[instrument]
 fn set_gauge_vec_cpu_time(gv: &prometheus::GaugeVec, cpu: &str, cpu_time: &procfs::CpuTime) {
     gv.with_label_values(&[cpu, "user"])
-        .set(cpu_time.user_ms() as f64);
+        .set(cpu_time.user as f64);
     gv.with_label_values(&[cpu, "nice"])
-        .set(cpu_time.nice_ms() as f64);
+        .set(cpu_time.nice as f64);
     gv.with_label_values(&[cpu, "system"])
-        .set(cpu_time.system_ms() as f64);
+        .set(cpu_time.system as f64);
     gv.with_label_values(&[cpu, "idle"])
-        .set(cpu_time.idle_ms() as f64);
+        .set(cpu_time.idle as f64);
     gv.with_label_values(&[cpu, "iowait"])
-        .set(cpu_time.iowait_ms().unwrap_or(0) as f64);
+        .set(cpu_time.iowait.unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "irq"])
-        .set(cpu_time.irq_ms().unwrap_or(0) as f64);
+        .set(cpu_time.irq.unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "softirq"])
-        .set(cpu_time.softirq_ms().unwrap_or(0) as f64);
+        .set(cpu_time.softirq.unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "steal"])
-        .set(cpu_time.steal_ms().unwrap_or(0) as f64);
+        .set(cpu_time.steal.unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "guest"])
-        .set(cpu_time.guest_ms().unwrap_or(0) as f64);
+        .set(cpu_time.guest.unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "guest_nice"])
-        .set(cpu_time.guest_nice_ms().unwrap_or(0) as f64);
+        .set(cpu_time.guest_nice.unwrap_or(0) as f64);
 }
 
 #[instrument]

src/agent/src/mount.rs

@@ -321,39 +321,26 @@ fn allocate_hugepages(logger: &Logger, options: &[String]) -> Result<()> {
 
     // sysfs entry is always of the form hugepages-${pagesize}kB
     // Ref: https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt
-    let path = Path::new(SYS_FS_HUGEPAGES_PREFIX)
-        .join(format!("hugepages-{}kB", pagesize / 1024))
-        .join("nr_hugepages");
+    let path = Path::new(SYS_FS_HUGEPAGES_PREFIX).join(format!("hugepages-{}kB", pagesize / 1024));
+
+    if !path.exists() {
+        fs::create_dir_all(&path).context("create hugepages-size directory")?;
+    }
 
     // write numpages to nr_hugepages file.
+    let path = path.join("nr_hugepages");
     let numpages = format!("{}", size / pagesize);
     info!(logger, "write {} pages to {:?}", &numpages, &path);
 
     let mut file = OpenOptions::new()
         .write(true)
+        .create(true)
         .open(&path)
         .context(format!("open nr_hugepages directory {:?}", &path))?;
 
     file.write_all(numpages.as_bytes())
         .context(format!("write nr_hugepages failed: {:?}", &path))?;
 
-    // Even if the write succeeds, the kernel isn't guaranteed to be
-    // able to allocate all the pages we requested.  Verify that it
-    // did.
-    let verify = fs::read_to_string(&path).context(format!("reading {:?}", &path))?;
-    let allocated = verify
-        .trim_end()
-        .parse::<u64>()
-        .map_err(|_| anyhow!("Unexpected text {:?} in {:?}", &verify, &path))?;
-    if allocated != size / pagesize {
-        return Err(anyhow!(
-            "Only allocated {} of {} hugepages of size {}",
-            allocated,
-            numpages,
-            pagesize
-        ));
-    }
-
     Ok(())
 }
 
@@ -1036,7 +1023,7 @@ mod tests {
             let dest_filename: String;
 
             if !d.src.is_empty() {
-                src = dir.path().join(d.src);
+                src = dir.path().join(d.src.to_string());
                 src_filename = src
                     .to_str()
                     .expect("failed to convert src to filename")
@@ -1046,7 +1033,7 @@ mod tests {
             }
 
             if !d.dest.is_empty() {
-                dest = dir.path().join(d.dest);
+                dest = dir.path().join(d.dest.to_string());
                 dest_filename = dest
                     .to_str()
                     .expect("failed to convert dest to filename")

src/agent/src/rpc.rs

@@ -85,8 +85,6 @@ use std::path::PathBuf;
 const CONTAINER_BASE: &str = "/run/kata-containers";
 const MODPROBE_PATH: &str = "/sbin/modprobe";
 
-const ERR_INVALID_BLOCK_SIZE: &str = "Invalid block size";
-
 // Convenience macro to obtain the scope logger
 macro_rules! sl {
     () => {
@@ -1221,12 +1219,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         info!(sl!(), "get guest details!");
         let mut resp = GuestDetailsResponse::new();
         // to get memory block size
-        match get_memory_info(
-            req.mem_block_size,
-            req.mem_hotplug_probe,
-            SYSFS_MEMORY_BLOCK_SIZE_PATH,
-            SYSFS_MEMORY_HOTPLUG_PROBE_PATH,
-        ) {
+        match get_memory_info(req.mem_block_size, req.mem_hotplug_probe) {
             Ok((u, v)) => {
                 resp.mem_block_size_bytes = u;
                 resp.support_mem_hotplug_probe = v;
@@ -1415,29 +1408,24 @@ impl protocols::health_ttrpc::Health for HealthService {
     }
 }
 
-fn get_memory_info(
-    block_size: bool,
-    hotplug: bool,
-    block_size_path: &str,
-    hotplug_probe_path: &str,
-) -> Result<(u64, bool)> {
+fn get_memory_info(block_size: bool, hotplug: bool) -> Result<(u64, bool)> {
     let mut size: u64 = 0;
     let mut plug: bool = false;
     if block_size {
-        match fs::read_to_string(block_size_path) {
+        match fs::read_to_string(SYSFS_MEMORY_BLOCK_SIZE_PATH) {
             Ok(v) => {
                 if v.is_empty() {
-                    warn!(sl!(), "file {} is empty", block_size_path);
-                    return Err(anyhow!(ERR_INVALID_BLOCK_SIZE));
+                    info!(sl!(), "string in empty???");
+                    return Err(anyhow!("Invalid block size"));
                 }
 
                 size = u64::from_str_radix(v.trim(), 16).map_err(|_| {
                     warn!(sl!(), "failed to parse the str {} to hex", size);
-                    anyhow!(ERR_INVALID_BLOCK_SIZE)
+                    anyhow!("Invalid block size")
                 })?;
             }
             Err(e) => {
-                warn!(sl!(), "memory block size error: {:?}", e.kind());
+                info!(sl!(), "memory block size error: {:?}", e.kind());
                 if e.kind() != std::io::ErrorKind::NotFound {
                     return Err(anyhow!(e));
                 }
@@ -1446,10 +1434,10 @@ fn get_memory_info(
     }
 
     if hotplug {
-        match stat::stat(hotplug_probe_path) {
+        match stat::stat(SYSFS_MEMORY_HOTPLUG_PROBE_PATH) {
             Ok(_) => plug = true,
             Err(e) => {
-                warn!(sl!(), "hotplug memory error: {:?}", e);
+                info!(sl!(), "hotplug memory error: {:?}", e);
                 match e {
                     nix::Error::ENOENT => plug = false,
                     _ => return Err(anyhow!(e)),
@@ -1880,35 +1868,8 @@ mod tests {
     use super::*;
     use crate::protocols::agent_ttrpc::AgentService as _;
     use oci::{Hook, Hooks};
-    use tempfile::tempdir;
     use ttrpc::{r#async::TtrpcContext, MessageHeader};
 
-    // Parameters:
-    //
-    // 1: expected Result
-    // 2: actual Result
-    // 3: string used to identify the test on error
-    macro_rules! assert_result {
-        ($expected_result:expr, $actual_result:expr, $msg:expr) => {
-            if $expected_result.is_ok() {
-                let expected_level = $expected_result.as_ref().unwrap();
-                let actual_level = $actual_result.unwrap();
-                assert!(*expected_level == actual_level, "{}", $msg);
-            } else {
-                let expected_error = $expected_result.as_ref().unwrap_err();
-                let expected_error_msg = format!("{:?}", expected_error);
-
-                if let Err(actual_error) = $actual_result {
-                    let actual_error_msg = format!("{:?}", actual_error);
-
-                    assert!(expected_error_msg == actual_error_msg, "{}", $msg);
-                } else {
-                    assert!(expected_error_msg == "expected error, got OK", "{}", $msg);
-                }
-            }
-        };
-    }
-
     fn mk_ttrpc_context() -> TtrpcContext {
         TtrpcContext {
             fd: -1,
@@ -2010,119 +1971,6 @@ mod tests {
         assert!(result.is_err(), "expected add arp neighbors to fail");
     }
 
-    #[tokio::test]
-    async fn test_get_memory_info() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            // if None is provided, no file will be generated, else the data in the Option will populate the file
-            block_size_data: Option<&'a str>,
-
-            hotplug_probe_data: bool,
-            get_block_size: bool,
-            get_hotplug: bool,
-            result: Result<(u64, bool)>,
-        }
-
-        let tests = &[
-            TestData {
-                block_size_data: Some("10000000"),
-                hotplug_probe_data: true,
-                get_block_size: true,
-                get_hotplug: true,
-                result: Ok((268435456, true)),
-            },
-            TestData {
-                block_size_data: Some("100"),
-                hotplug_probe_data: false,
-                get_block_size: true,
-                get_hotplug: true,
-                result: Ok((256, false)),
-            },
-            TestData {
-                block_size_data: None,
-                hotplug_probe_data: false,
-                get_block_size: true,
-                get_hotplug: true,
-                result: Ok((0, false)),
-            },
-            TestData {
-                block_size_data: Some(""),
-                hotplug_probe_data: false,
-                get_block_size: true,
-                get_hotplug: false,
-                result: Err(anyhow!(ERR_INVALID_BLOCK_SIZE)),
-            },
-            TestData {
-                block_size_data: Some("-1"),
-                hotplug_probe_data: false,
-                get_block_size: true,
-                get_hotplug: false,
-                result: Err(anyhow!(ERR_INVALID_BLOCK_SIZE)),
-            },
-            TestData {
-                block_size_data: Some("    "),
-                hotplug_probe_data: false,
-                get_block_size: true,
-                get_hotplug: false,
-                result: Err(anyhow!(ERR_INVALID_BLOCK_SIZE)),
-            },
-            TestData {
-                block_size_data: Some("some data"),
-                hotplug_probe_data: false,
-                get_block_size: true,
-                get_hotplug: false,
-                result: Err(anyhow!(ERR_INVALID_BLOCK_SIZE)),
-            },
-            TestData {
-                block_size_data: Some("some data"),
-                hotplug_probe_data: true,
-                get_block_size: false,
-                get_hotplug: false,
-                result: Ok((0, false)),
-            },
-            TestData {
-                block_size_data: Some("100"),
-                hotplug_probe_data: true,
-                get_block_size: false,
-                get_hotplug: false,
-                result: Ok((0, false)),
-            },
-            TestData {
-                block_size_data: Some("100"),
-                hotplug_probe_data: true,
-                get_block_size: false,
-                get_hotplug: true,
-                result: Ok((0, true)),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let dir = tempdir().expect("failed to make tempdir");
-            let block_size_path = dir.path().join("block_size_bytes");
-            let hotplug_probe_path = dir.path().join("probe");
-
-            if let Some(block_size_data) = d.block_size_data {
-                fs::write(&block_size_path, block_size_data).unwrap();
-            }
-            if d.hotplug_probe_data {
-                fs::write(&hotplug_probe_path, []).unwrap();
-            }
-
-            let result = get_memory_info(
-                d.get_block_size,
-                d.get_hotplug,
-                block_size_path.to_str().unwrap(),
-                hotplug_probe_path.to_str().unwrap(),
-            );
-
-            let msg = format!("{}, result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
-    }
-
     #[tokio::test]
     async fn test_verify_cid() {
         #[derive(Debug)]

src/agent/src/signal.rs

@@ -58,16 +58,17 @@ async fn handle_sigchild(logger: Logger, sandbox: Arc<Mutex<Sandbox>>) -> Result
             }
 
             let mut p = process.unwrap();
+            let ret: i32;
 
-            let ret: i32 = match wait_status {
-                WaitStatus::Exited(_, c) => c,
-                WaitStatus::Signaled(_, sig, _) => sig as i32,
+            match wait_status {
+                WaitStatus::Exited(_, c) => ret = c,
+                WaitStatus::Signaled(_, sig, _) => ret = sig as i32,
                 _ => {
                     info!(logger, "got wrong status for process";
                                   "child-status" => format!("{:?}", wait_status));
                     continue;
                 }
-            };
+            }
 
             p.exit_code = ret;
             let _ = p.exit_tx.take();

src/agent/src/util.rs

@@ -237,6 +237,8 @@ mod tests {
                 JoinError,
             >;
 
+            let result: std::result::Result<u64, std::io::Error>;
+
             select! {
                 res = handle => spawn_result = res,
                 _ = &mut timeout => panic!("timed out"),
@@ -244,7 +246,7 @@ mod tests {
 
             assert!(spawn_result.is_ok());
 
-            let result: std::result::Result<u64, std::io::Error> = spawn_result.unwrap();
+            result = spawn_result.unwrap();
 
             assert!(result.is_ok());
 
@@ -276,6 +278,8 @@ mod tests {
 
         let spawn_result: std::result::Result<std::result::Result<u64, std::io::Error>, JoinError>;
 
+        let result: std::result::Result<u64, std::io::Error>;
+
         select! {
             res = handle => spawn_result = res,
             _ = &mut timeout => panic!("timed out"),
@@ -283,7 +287,7 @@ mod tests {
 
         assert!(spawn_result.is_ok());
 
-        let result: std::result::Result<u64, std::io::Error> = spawn_result.unwrap();
+        result = spawn_result.unwrap();
 
         assert!(result.is_ok());
 
@@ -316,6 +320,8 @@ mod tests {
 
         let spawn_result: std::result::Result<std::result::Result<u64, std::io::Error>, JoinError>;
 
+        let result: std::result::Result<u64, std::io::Error>;
+
         select! {
             res = handle => spawn_result = res,
             _ = &mut timeout => panic!("timed out"),
@@ -323,7 +329,7 @@ mod tests {
 
         assert!(spawn_result.is_ok());
 
-        let result: std::result::Result<u64, std::io::Error> = spawn_result.unwrap();
+        result = spawn_result.unwrap();
 
         assert!(result.is_ok());
 

src/agent/vsock-exporter/src/lib.rs

@@ -178,11 +178,13 @@ impl Builder {
     pub fn init(self) -> Exporter {
         let Builder { port, cid, logger } = self;
 
-        let cid_str: String = if self.cid == libc::VMADDR_CID_ANY {
-            ANY_CID.to_string()
+        let cid_str: String;
+
+        if self.cid == libc::VMADDR_CID_ANY {
+            cid_str = ANY_CID.to_string();
         } else {
-            format!("{}", self.cid)
-        };
+            cid_str = format!("{}", self.cid);
+        }
 
         Exporter {
             port,

src/libs/protocols/hack/update-generated-proto.sh

@@ -7,14 +7,14 @@
 # //
 
 die() {
-    cat <<EOF >&2
+    cat <<EOT >&2
 ====================================================================
 ====                compile protocols failed                    ====
 
 $1
 
 ====================================================================
-EOF
+EOT
     exit 1
 }
 

src/runtime/Makefile

@@ -592,7 +592,7 @@ generate-config: $(CONFIGS)
 test: hook go-test
 
 hook:
-	make -C pkg/katautils/mockhook
+	make -C virtcontainers hook
 
 go-test: $(GENERATED_FILES)
 	go clean -testcache

src/runtime/README.md

@@ -158,7 +158,7 @@ See [the community repository](https://github.com/kata-containers/community).
 
 ### Contact
 
-See [how to reach the community](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md#contact).
+See [how to reach the community](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md#contact).
 
 ## Further information
 

src/runtime/pkg/katautils/hook_test.go

@@ -20,7 +20,7 @@ import (
 var testKeyHook = "test-key"
 var testContainerIDHook = "test-container-id"
 var testControllerIDHook = "test-controller-id"
-var testBinHookPath = "mockhook/hook"
+var testBinHookPath = "../../virtcontainers/hook/mock/hook"
 var testBundlePath = "/test/bundle"
 
 func getMockHookBinPath() string {

src/runtime/pkg/katautils/mockhook/.gitignore

@@ -1 +0,0 @@
-hook

src/runtime/pkg/katautils/mockhook/Makefile

@@ -1,21 +0,0 @@
-# Copyright Red Hat.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-BIN = hook
-SRC = hook.go
-
-V              = @
-Q              = $(V:1=)
-QUIET_BUILD    = $(Q:@=@echo    '     BUILD    '$@;)
-
-BUILDFLAGS     =
-
-all: $(BIN)
-
-$(BIN): $(SRC)
-	$(QUIET_BUILD)go build $(BUILDFLAGS) -o $@ $^
-
-clean:
-	rm -f $(BIN)

src/runtime/virtcontainers/Makefile

@@ -5,6 +5,11 @@
 #
 
 PREFIX := /usr
+BIN_DIR := $(PREFIX)/bin
+VC_BIN_DIR := $(BIN_DIR)/virtcontainers/bin
+TEST_BIN_DIR := $(VC_BIN_DIR)/test
+HOOK_DIR := hook/mock
+HOOK_BIN := hook
 MK_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 GOBUILD_FLAGS := -mod=vendor
 
@@ -20,11 +25,16 @@ QUIET_GOBUILD = $(Q:@=@echo    '     GOBUILD  '$@;)
 # Build
 #
 
-all: build
+all: build binaries
 
 build:
 	$(QUIET_GOBUILD)go build $(GOBUILD_FLAGS) $(go list ./... | grep -v /vendor/)
 
+hook:
+	$(QUIET_GOBUILD)go build $(GOBUILD_FLAGS) -o $(HOOK_DIR)/$@ $(HOOK_DIR)/*.go
+
+binaries: hook
+
 #
 # Tests
 #
@@ -35,7 +45,40 @@ check-go-static:
 	bash $(MK_DIR)/../../../ci/static-checks.sh
 
 check-go-test:
-	bash $(MK_DIR)/../../../ci/go-test.sh
+	bash $(MK_DIR)/../../../ci/go-test.sh \
+		$(TEST_BIN_DIR)/$(HOOK_BIN)
+
+#
+# Install
+#
+
+define INSTALL_EXEC
+	install -D $1 $(VC_BIN_DIR)/ || exit 1;
+endef
+
+define INSTALL_TEST_EXEC
+	install -D $1 $(TEST_BIN_DIR)/ || exit 1;
+endef
+
+install:
+	@mkdir -p $(VC_BIN_DIR)
+	@mkdir -p $(TEST_BIN_DIR)
+	$(call INSTALL_TEST_EXEC,$(HOOK_DIR)/$(HOOK_BIN))
+
+#
+# Uninstall
+#
+
+define UNINSTALL_EXEC
+	rm -f $(call FILE_SAFE_TO_REMOVE,$(VC_BIN_DIR)/$1) || exit 1;
+endef
+
+define UNINSTALL_TEST_EXEC
+	rm -f $(call FILE_SAFE_TO_REMOVE,$(TEST_BIN_DIR)/$1) || exit 1;
+endef
+
+uninstall:
+	$(call UNINSTALL_TEST_EXEC,$(HOOK_BIN))
 
 #
 # Clean
@@ -47,7 +90,7 @@ define FILE_SAFE_TO_REMOVE =
 $(shell test -e "$(1)" && test "$(1)" != "/" && echo "$(1)")
 endef
 
-CLEAN_FILES +=
+CLEAN_FILES += $(HOOK_DIR)/$(HOOK_BIN)
 
 clean:
 	rm -f $(foreach f,$(CLEAN_FILES),$(call FILE_SAFE_TO_REMOVE,$(f)))
@@ -55,7 +98,11 @@ clean:
 .PHONY: \
 	all \
 	build \
+	hook \
+	binaries \
 	check \
 	check-go-static \
 	check-go-test \
+	install \
+	uninstall \
 	clean

src/runtime/virtcontainers/container_test.go

@@ -86,7 +86,7 @@ func TestContainerRemoveDrive(t *testing.T) {
 	sandbox := &Sandbox{
 		ctx:        context.Background(),
 		id:         "sandbox",
-		devManager: manager.NewDeviceManager(config.VirtioSCSI, false, "", nil),
+		devManager: manager.NewDeviceManager(manager.VirtioSCSI, false, "", nil),
 		config:     &SandboxConfig{},
 	}
 
@@ -320,7 +320,7 @@ func TestContainerAddDriveDir(t *testing.T) {
 	sandbox := &Sandbox{
 		ctx:        context.Background(),
 		id:         testSandboxID,
-		devManager: manager.NewDeviceManager(config.VirtioSCSI, false, "", nil),
+		devManager: manager.NewDeviceManager(manager.VirtioSCSI, false, "", nil),
 		hypervisor: &mockHypervisor{},
 		agent:      &mockAgent{},
 		config: &SandboxConfig{

src/runtime/virtcontainers/device/config/config.go

@@ -51,7 +51,7 @@ const (
 	// VirtioBlock means use virtio-blk for hotplugging drives
 	VirtioBlock = "virtio-blk"
 
-	// VirtioBlockCCW means use virtio-blk-ccw for hotplugging drives
+	// VirtioBlockCCW means use virtio-blk for hotplugging drives
 	VirtioBlockCCW = "virtio-blk-ccw"
 
 	// VirtioSCSI means use virtio-scsi for hotplugging drives
@@ -72,12 +72,6 @@ const (
 	VirtioFSNydus = "virtio-fs-nydus"
 )
 
-const (
-	// Define the string key for DriverOptions in DeviceInfo struct
-	FsTypeOpt      = "fstype"
-	BlockDriverOpt = "block-driver"
-)
-
 const (
 	// The OCI spec requires the major-minor number to be provided for a
 	// device. We have chosen the below major numbers to represent
@@ -103,7 +97,7 @@ var getSysDevPath = getSysDevPathImpl
 // DeviceInfo is an embedded type that contains device data common to all types of devices.
 type DeviceInfo struct {
 	// DriverOptions is specific options for each device driver
-	// for example, for BlockDevice, we can set DriverOptions["block-driver"]="virtio-blk"
+	// for example, for BlockDevice, we can set DriverOptions["blockDriver"]="virtio-blk"
 	DriverOptions map[string]string
 
 	// Hostpath is device path on host

src/runtime/virtcontainers/device/config/pmem.go

@@ -81,8 +81,8 @@ func PmemDeviceInfo(source, destination string) (*DeviceInfo, error) {
 		fstype = "ext4"
 	}
 
-	pmemLog.WithField(FsTypeOpt, fstype).Debug("filesystem for mount point")
-	device.DriverOptions[FsTypeOpt] = fstype
+	pmemLog.WithField("fstype", fstype).Debug("filesystem for mount point")
+	device.DriverOptions["fstype"] = fstype
 
 	return device, nil
 }

src/runtime/virtcontainers/device/drivers/block.go

@@ -70,13 +70,13 @@ func (device *BlockDevice) Attach(ctx context.Context, devReceiver api.DeviceRec
 		ReadOnly: device.DeviceInfo.ReadOnly,
 	}
 
-	if fs, ok := device.DeviceInfo.DriverOptions[config.FsTypeOpt]; ok {
+	if fs, ok := device.DeviceInfo.DriverOptions["fstype"]; ok {
 		drive.Format = fs
 	}
 
 	customOptions := device.DeviceInfo.DriverOptions
 	if customOptions == nil ||
-		customOptions[config.BlockDriverOpt] == config.VirtioSCSI {
+		customOptions["block-driver"] == "virtio-scsi" {
 		// User has not chosen a specific block device type
 		// Default to SCSI
 		scsiAddr, err := utils.GetSCSIAddress(index)
@@ -85,15 +85,15 @@ func (device *BlockDevice) Attach(ctx context.Context, devReceiver api.DeviceRec
 		}
 
 		drive.SCSIAddr = scsiAddr
-	} else if customOptions[config.BlockDriverOpt] != config.Nvdimm {
+	} else if customOptions["block-driver"] != "nvdimm" {
 		var globalIdx int
 
-		switch customOptions[config.BlockDriverOpt] {
-		case config.VirtioBlock:
+		switch customOptions["block-driver"] {
+		case "virtio-blk":
 			globalIdx = index
-		case config.VirtioBlockCCW:
+		case "virtio-blk-ccw":
 			globalIdx = index
-		case config.VirtioMmio:
+		case "virtio-mmio":
 			//With firecracker the rootfs for the VM itself
 			//sits at /dev/vda and consumes the first index.
 			//Longer term block based VM rootfs should be added
@@ -111,7 +111,7 @@ func (device *BlockDevice) Attach(ctx context.Context, devReceiver api.DeviceRec
 		drive.VirtPath = filepath.Join("/dev", driveName)
 	}
 
-	deviceLogger().WithField("device", device.DeviceInfo.HostPath).WithField("VirtPath", drive.VirtPath).Infof("Attaching %s device", customOptions[config.BlockDriverOpt])
+	deviceLogger().WithField("device", device.DeviceInfo.HostPath).WithField("VirtPath", drive.VirtPath).Infof("Attaching %s device", customOptions["block-driver"])
 	device.BlockDrive = drive
 	if err = devReceiver.HotplugAddDevice(ctx, device, config.DeviceBlock); err != nil {
 		return err

src/runtime/virtcontainers/device/drivers/vhost_user_blk.go

@@ -100,14 +100,14 @@ func isVirtioBlkBlockDriver(customOptions map[string]string) bool {
 	if customOptions == nil {
 		// User has not chosen a specific block device type
 		// Default to SCSI
-		blockDriverOption = config.VirtioSCSI
+		blockDriverOption = "virtio-scsi"
 	} else {
-		blockDriverOption = customOptions[config.BlockDriverOpt]
+		blockDriverOption = customOptions["block-driver"]
 	}
 
-	if blockDriverOption == config.VirtioBlock ||
-		blockDriverOption == config.VirtioBlockCCW ||
-		blockDriverOption == config.VirtioMmio {
+	if blockDriverOption == "virtio-blk" ||
+		blockDriverOption == "virtio-blk-ccw" ||
+		blockDriverOption == "virtio-mmio" {
 		return true
 	}
 

src/runtime/virtcontainers/device/manager/manager.go

@@ -21,6 +21,19 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
+const (
+	// VirtioMmio indicates block driver is virtio-mmio based
+	VirtioMmio string = "virtio-mmio"
+	// VirtioBlock indicates block driver is virtio-blk based
+	VirtioBlock string = "virtio-blk"
+	// VirtioBlockCCW indicates block driver is virtio-blk-ccw based
+	VirtioBlockCCW string = "virtio-blk-ccw"
+	// VirtioSCSI indicates block driver is virtio-scsi based
+	VirtioSCSI string = "virtio-scsi"
+	// Nvdimm indicates block driver is nvdimm based
+	Nvdimm string = "nvdimm"
+)
+
 var (
 	// ErrIDExhausted represents that devices are too many
 	// and no more IDs can be generated
@@ -56,16 +69,16 @@ func NewDeviceManager(blockDriver string, vhostUserStoreEnabled bool, vhostUserS
 		vhostUserStorePath:    vhostUserStorePath,
 		devices:               make(map[string]api.Device),
 	}
-	if blockDriver == config.VirtioMmio {
-		dm.blockDriver = config.VirtioMmio
-	} else if blockDriver == config.VirtioBlock {
-		dm.blockDriver = config.VirtioBlock
-	} else if blockDriver == config.Nvdimm {
-		dm.blockDriver = config.Nvdimm
-	} else if blockDriver == config.VirtioBlockCCW {
-		dm.blockDriver = config.VirtioBlockCCW
+	if blockDriver == VirtioMmio {
+		dm.blockDriver = VirtioMmio
+	} else if blockDriver == VirtioBlock {
+		dm.blockDriver = VirtioBlock
+	} else if blockDriver == Nvdimm {
+		dm.blockDriver = Nvdimm
+	} else if blockDriver == VirtioBlockCCW {
+		dm.blockDriver = VirtioBlockCCW
 	} else {
-		dm.blockDriver = config.VirtioSCSI
+		dm.blockDriver = VirtioSCSI
 	}
 
 	drivers.AllPCIeDevs = make(map[string]bool)
@@ -119,13 +132,13 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device
 		if devInfo.DriverOptions == nil {
 			devInfo.DriverOptions = make(map[string]string)
 		}
-		devInfo.DriverOptions[config.BlockDriverOpt] = dm.blockDriver
+		devInfo.DriverOptions["block-driver"] = dm.blockDriver
 		return drivers.NewVhostUserBlkDevice(&devInfo), nil
 	} else if isBlock(devInfo) {
 		if devInfo.DriverOptions == nil {
 			devInfo.DriverOptions = make(map[string]string)
 		}
-		devInfo.DriverOptions[config.BlockDriverOpt] = dm.blockDriver
+		devInfo.DriverOptions["block-driver"] = dm.blockDriver
 		return drivers.NewBlockDevice(&devInfo), nil
 	} else {
 		deviceLogger().WithField("device", devInfo.HostPath).Info("Device has not been passed to the container")

src/runtime/virtcontainers/device/manager/manager_linux_test.go

@@ -31,7 +31,7 @@ func TestAttachVhostUserBlkDevice(t *testing.T) {
 
 	tmpDir, err := os.MkdirTemp("", "")
 	dm := &deviceManager{
-		blockDriver:           config.VirtioBlock,
+		blockDriver:           VirtioBlock,
 		devices:               make(map[string]api.Device),
 		vhostUserStoreEnabled: true,
 		vhostUserStorePath:    tmpDir,

src/runtime/virtcontainers/device/manager/manager_test.go

@@ -26,7 +26,7 @@ const dirMode = os.FileMode(0750) | os.ModeDir
 
 func TestNewDevice(t *testing.T) {
 	dm := &deviceManager{
-		blockDriver: config.VirtioBlock,
+		blockDriver: VirtioBlock,
 		devices:     make(map[string]api.Device),
 	}
 	savedSysDevPrefix := config.SysDevPrefix
@@ -96,7 +96,7 @@ func TestNewDevice(t *testing.T) {
 
 func TestAttachVFIODevice(t *testing.T) {
 	dm := &deviceManager{
-		blockDriver: config.VirtioBlock,
+		blockDriver: VirtioBlock,
 		devices:     make(map[string]api.Device),
 	}
 	tmpDir, err := os.MkdirTemp("", "")
@@ -155,7 +155,7 @@ func TestAttachVFIODevice(t *testing.T) {
 
 func TestAttachGenericDevice(t *testing.T) {
 	dm := &deviceManager{
-		blockDriver: config.VirtioBlock,
+		blockDriver: VirtioBlock,
 		devices:     make(map[string]api.Device),
 	}
 	path := "/dev/tty2"
@@ -180,7 +180,7 @@ func TestAttachGenericDevice(t *testing.T) {
 
 func TestAttachBlockDevice(t *testing.T) {
 	dm := &deviceManager{
-		blockDriver: config.VirtioBlock,
+		blockDriver: VirtioBlock,
 		devices:     make(map[string]api.Device),
 	}
 	path := "/dev/hda"
@@ -203,7 +203,7 @@ func TestAttachBlockDevice(t *testing.T) {
 	assert.Nil(t, err)
 
 	// test virtio SCSI driver
-	dm.blockDriver = config.VirtioSCSI
+	dm.blockDriver = VirtioSCSI
 	device, err = dm.NewDevice(deviceInfo)
 	assert.Nil(t, err)
 	err = device.Attach(context.Background(), devReceiver)
@@ -214,7 +214,7 @@ func TestAttachBlockDevice(t *testing.T) {
 }
 
 func TestAttachDetachDevice(t *testing.T) {
-	dm := NewDeviceManager(config.VirtioSCSI, false, "", nil)
+	dm := NewDeviceManager(VirtioSCSI, false, "", nil)
 
 	path := "/dev/hda"
 	deviceInfo := config.DeviceInfo{

src/runtime/virtcontainers/documentation/api/1.0/api.md

@@ -547,7 +547,7 @@ type DeviceInfo struct {
 	ID string
 
 	// DriverOptions is specific options for each device driver
-	// for example, for BlockDevice, we can set DriverOptions["block-driver"]="virtio-blk"
+	// for example, for BlockDevice, we can set DriverOptions["blockDriver"]="virtio-blk"
 	DriverOptions map[string]string
 }
 ```
@@ -835,7 +835,7 @@ type DeviceInfo struct {
 	ID string
 
 	// DriverOptions is specific options for each device driver
-	// for example, for BlockDevice, we can set DriverOptions["block-driver"]="virtio-blk"
+	// for example, for BlockDevice, we can set DriverOptions["blockDriver"]="virtio-blk"
 	DriverOptions map[string]string
 }
 ```

src/runtime/virtcontainers/kata_agent_test.go

@@ -390,10 +390,10 @@ func TestHandleBlockVolume(t *testing.T) {
 	mounts = append(mounts, vMount, bMount, dMount)
 
 	tmpDir := "/vhost/user/dir"
-	dm := manager.NewDeviceManager(config.VirtioBlock, true, tmpDir, devices)
+	dm := manager.NewDeviceManager(manager.VirtioBlock, true, tmpDir, devices)
 
 	sConfig := SandboxConfig{}
-	sConfig.HypervisorConfig.BlockDeviceDriver = config.VirtioBlock
+	sConfig.HypervisorConfig.BlockDeviceDriver = manager.VirtioBlock
 	sandbox := Sandbox{
 		id:         "100",
 		containers: containers,

src/runtime/virtcontainers/persist/api/device.go

@@ -86,7 +86,7 @@ type VhostUserDeviceAttrs struct {
 // Refs: virtcontainers/device/drivers/generic.go:GenericDevice
 type DeviceState struct {
 	// DriverOptions is specific options for each device driver
-	// for example, for BlockDevice, we can set DriverOptions["block-driver"]="virtio-blk"
+	// for example, for BlockDevice, we can set DriverOptions["blockDriver"]="virtio-blk"
 	DriverOptions map[string]string
 
 	// VhostUserDeviceAttrs is specific for vhost-user device driver

src/runtime/virtcontainers/persist_test.go

@@ -10,11 +10,11 @@ import (
 	"os"
 	"testing"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/stretchr/testify/assert"
+
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
-	"github.com/stretchr/testify/assert"
 )
 
 func TestSandboxRestore(t *testing.T) {
@@ -32,7 +32,7 @@ func TestSandboxRestore(t *testing.T) {
 	sandbox := Sandbox{
 		id:         "test-exp",
 		containers: container,
-		devManager: manager.NewDeviceManager(config.VirtioSCSI, false, "", nil),
+		devManager: manager.NewDeviceManager(manager.VirtioSCSI, false, "", nil),
 		hypervisor: &mockHypervisor{},
 		network:    network,
 		ctx:        context.Background(),

src/runtime/virtcontainers/qemu.go

@@ -758,7 +758,7 @@ func (q *qemu) setupVirtioMem(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	// backend memory size must be multiple of 4Mib
+	// backend memory size must be multiple of 2Mib
 	sizeMB := (int(maxMem) - int(q.config.MemorySize)) >> 2 << 2
 
 	share, target, memoryBack, err := q.getMemArgs()
@@ -783,6 +783,7 @@ func (q *qemu) setupVirtioMem(ctx context.Context) error {
 
 	err = q.qmpMonitorCh.qmp.ExecMemdevAdd(q.qmpMonitorCh.ctx, memoryBack, "virtiomem", target, sizeMB, share, "virtio-mem-pci", "virtiomem0", addr, bridge.ID)
 	if err == nil {
+		q.config.VirtioMem = true
 		q.Logger().Infof("Setup %dMB virtio-mem-pci success", sizeMB)
 	} else {
 		help := ""

src/runtime/virtcontainers/sandbox_test.go

@@ -548,7 +548,7 @@ func TestSandboxAttachDevicesVFIO(t *testing.T) {
 		config.SysIOMMUPath = savedIOMMUPath
 	}()
 
-	dm := manager.NewDeviceManager(config.VirtioSCSI, false, "", nil)
+	dm := manager.NewDeviceManager(manager.VirtioSCSI, false, "", nil)
 	path := filepath.Join(vfioPath, testFDIOGroup)
 	deviceInfo := config.DeviceInfo{
 		HostPath:      path,
@@ -599,7 +599,7 @@ func TestSandboxAttachDevicesVhostUserBlk(t *testing.T) {
 	tmpDir, err := os.MkdirTemp("", "")
 	assert.Nil(t, err)
 	os.RemoveAll(tmpDir)
-	dm := manager.NewDeviceManager(config.VirtioSCSI, true, tmpDir, nil)
+	dm := manager.NewDeviceManager(manager.VirtioSCSI, true, tmpDir, nil)
 
 	vhostUserDevNodePath := filepath.Join(tmpDir, "/block/devices/")
 	vhostUserSockPath := filepath.Join(tmpDir, "/block/sockets/")

tools/osbuilder/dockerfiles/QAT/Dockerfile

@@ -16,7 +16,7 @@ ENV QAT_DRIVER_URL "https://downloadmirror.intel.com/649693/${QAT_DRIVER_VER}"
 ENV QAT_CONFIGURE_OPTIONS "--enable-icp-sriov=guest"
 ENV KATA_REPO_VERSION "main"
 ENV AGENT_VERSION ""
-ENV ROOTFS_OS "centos"
+ENV ROOTFS_OS "ubuntu"
 ENV OUTPUT_DIR "/output"
 
 RUN dnf install -y \

tools/osbuilder/dockerfiles/QAT/run.sh

@@ -39,9 +39,9 @@ grab_kata_repos()
     # Check out all the repos we will use now, so we can try and ensure they use the specified branch
     # Only check out the branch needed, and make it shallow and thus space/bandwidth efficient
     # Use a green prompt with white text for easy viewing
-    /bin/echo -e "\n\e[1;42mClone and checkout Kata repos\e[0m"
-    [ -d "${kata_repo_path}" ] || git clone --single-branch --branch $KATA_REPO_VERSION --depth=1 https://${kata_repo} ${kata_repo_path}
-    [ -d "${tests_repo_path}" ] || git clone --single-branch --branch $KATA_REPO_VERSION --depth=1 https://${tests_repo} ${tests_repo_path}
+    /bin/echo -e "\n\e[1;42mClone and checkout Kata repos\e[0m" 
+    git clone --single-branch --branch $KATA_REPO_VERSION --depth=1 https://${kata_repo} ${kata_repo_path}
+    git clone --single-branch --branch $KATA_REPO_VERSION --depth=1 https://${tests_repo} ${tests_repo_path}
 }
 
 configure_kernel()
@@ -164,7 +164,6 @@ main()
 	done
 	shift $((OPTIND-1))
 
-	sudo chown -R qatbuilder:qatbuilder /home/qatbuilder
 	grab_qat_drivers
 	grab_kata_repos
 	configure_kernel

tools/osbuilder/dracut/Dockerfile.in

@@ -36,4 +36,7 @@ RUN zypper --non-interactive refresh; \
     zypper --non-interactive clean --all;
 
 
+# This will install the proper golang to build Kata components
+@INSTALL_MUSL@
+@INSTALL_GO@
 @INSTALL_RUST@

tools/osbuilder/image-builder/image_builder.sh

@@ -72,7 +72,7 @@ readonly mem_boundary_mb=128
 source "${lib_file}"
 
 usage() {
-	cat <<EOF
+	cat <<EOT
 Usage: ${script_name} [options] <rootfs-dir>
 	This script will create a Kata Containers image file of
 	an adequate size based on the <rootfs-dir> directory.
@@ -117,7 +117,7 @@ Kernels and hypervisors that support DAX/NVDIMM read the MBR #2, otherwise MBR #
 [1] - https://github.com/kata-containers/kata-containers/blob/main/tools/osbuilder/image-builder/nsdax.gpl.c
 [2] - https://github.com/torvalds/linux/blob/master/drivers/nvdimm/pfn.h
 
-EOF
+EOT
 }
 
 

tools/osbuilder/initrd-builder/initrd_builder.sh

@@ -23,7 +23,7 @@ AGENT_INIT=${AGENT_INIT:-no}
 usage()
 {
 	error="${1:-0}"
-	cat <<EOF
+	cat <<EOT
 Usage: ${script_name} [options] <rootfs-dir>
 	This script creates a Kata Containers initrd image file based on the
 	<rootfs-dir> directory.
@@ -38,7 +38,7 @@ Extra environment variables:
 		    DEFAULT: kata-agent
 	AGENT_INIT: use kata agent as init process
 		    DEFAULT: no
-EOF
+EOT
 exit "${error}"
 }
 

tools/osbuilder/rootfs-builder/alpine/Dockerfile.in

@@ -6,7 +6,7 @@
 ARG IMAGE_REGISTRY=docker.io
 FROM ${IMAGE_REGISTRY}/alpine:3.15
 
-RUN apk update && apk add --no-cache \
+RUN apk update && apk add \
     apk-tools-static \
     autoconf \
     automake \
@@ -26,5 +26,3 @@ RUN apk update && apk add --no-cache \
     musl-dev \
     protoc \
     tar
-# aarch64 requires this name -- link for all
-RUN ln -s /usr/bin/gcc "/usr/bin/$(uname -m)-linux-musl-gcc"

tools/osbuilder/rootfs-builder/clearlinux/Dockerfile.in

@@ -37,4 +37,5 @@ RUN dnf -y update && dnf install -y \
     dnf clean all
 
 # This will install the proper packages to build Kata components
+@INSTALL_MUSL@
 @INSTALL_RUST@

tools/osbuilder/rootfs-builder/debian/Dockerfile-aarch64.in

@@ -0,0 +1,34 @@
+#
+# Copyright (c) 2020 ARM Limited
+#
+# SPDX-License-Identifier: Apache-2.0
+
+ARG IMAGE_REGISTRY=docker.io
+# NOTE: OS_VERSION is set according to config.sh
+FROM ${IMAGE_REGISTRY}/debian:@OS_VERSION@
+
+# RUN commands
+RUN apt-get update && apt-get install -y \
+    autoconf \
+    automake \
+    binutils \
+    build-essential \
+    chrony \
+    coreutils \
+    curl \
+    debianutils \
+    debootstrap \
+    g++ \
+    gcc \
+    git \
+    libc-dev \
+    libstdc++-8-dev \
+    m4 \
+    make \
+    sed \
+    systemd \
+    tar \
+    vim
+# This will install the proper packages to build Kata components
+@INSTALL_MUSL@
+@INSTALL_RUST@

tools/osbuilder/rootfs-builder/debian/Dockerfile.in

@@ -27,14 +27,14 @@ RUN apt-get update && apt-get --no-install-recommends install -y \
     libstdc++-8-dev \
     m4 \
     make \
+    musl \
+    musl-dev \
     musl-tools \
     sed \
     systemd \
     tar \
     vim \
     wget
-# aarch64 requires this name -- link for all
-RUN ln -s /usr/bin/musl-gcc "/usr/bin/$(uname -m)-linux-musl-gcc"
 
 # This will install the proper packages to build Kata components
 @INSTALL_RUST@

tools/osbuilder/rootfs-builder/rootfs.sh

@@ -14,6 +14,7 @@ script_name="${0##*/}"
 script_dir="$(dirname $(readlink -f $0))"
 AGENT_VERSION=${AGENT_VERSION:-}
 RUST_VERSION="null"
+MUSL_VERSION=${MUSL_VERSION:-"null"}
 AGENT_BIN=${AGENT_BIN:-kata-agent}
 AGENT_INIT=${AGENT_INIT:-no}
 KERNEL_MODULES_DIR=${KERNEL_MODULES_DIR:-""}
@@ -39,11 +40,7 @@ handle_error() {
 trap 'handle_error $LINENO' ERR
 
 # Default architecture
-export ARCH=${ARCH:-$(uname -m)}
-if [ "$ARCH" == "ppc64le" ] || [ "$ARCH" == "s390x" ]; then
-	LIBC=gnu
-	echo "WARNING: Forcing LIBC=gnu because $ARCH has no musl Rust target"
-fi
+ARCH=$(uname -m)
 
 # distro-specific config file
 typeset -r CONFIG_SH="config.sh"
@@ -66,7 +63,7 @@ typeset init=
 usage()
 {
 	error="${1:-0}"
-	cat <<EOF
+	cat <<EOT
 
 Usage: ${script_name} [options] [DISTRO]
 
@@ -107,11 +104,6 @@ AGENT_SOURCE_BIN    Path to the directory of agent binary.
 AGENT_VERSION       Version of the agent to include in the rootfs.
                     Default value: ${AGENT_VERSION:-<not set>}
 
-ARCH                Target architecture (according to \`uname -m\`).
-                    Foreign bootstraps are currently only supported for Ubuntu
-                    and glibc agents.
-                    Default value: $(uname -m)
-
 DISTRO_REPO         Use host repositories to install guest packages.
                     Default value: <not set>
 
@@ -133,9 +125,6 @@ KERNEL_MODULES_DIR  Path to a directory containing kernel modules to include in
                     the rootfs.
                     Default value: <empty>
 
-LIBC                libc the agent is built against (gnu or musl).
-                    Default value: ${LIBC} (varies with architecture)
-
 ROOTFS_DIR          Path to the directory that is populated with the rootfs.
                     Default value: <${script_name} path>/rootfs-<DISTRO-name>
 
@@ -154,7 +143,7 @@ Refer to the Platform-OS Compatibility Matrix for more details on the supported
 architectures:
 https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder#platform-distro-compatibility-matrix
 
-EOF
+EOT
 exit "${error}"
 }
 
@@ -346,6 +335,11 @@ build_rootfs_distro()
 
 	echo "Required rust version: $RUST_VERSION"
 
+	detect_musl_version ||
+		die "Could not detect the required musl version for AGENT_VERSION='${AGENT_VERSION:-main}'."
+
+	echo "Required musl version: $MUSL_VERSION"
+
 	if [ -z "${USE_DOCKER}" ] && [ -z "${USE_PODMAN}" ]; then
 		info "build directly"
 		build_rootfs ${ROOTFS_DIR}
@@ -417,10 +411,8 @@ build_rootfs_distro()
 			--env ROOTFS_DIR="/rootfs" \
 			--env AGENT_BIN="${AGENT_BIN}" \
 			--env AGENT_INIT="${AGENT_INIT}" \
-			--env ARCH="${ARCH}" \
 			--env CI="${CI}" \
 			--env KERNEL_MODULES_DIR="${KERNEL_MODULES_DIR}" \
-			--env LIBC="${LIBC}" \
 			--env EXTRA_PKGS="${EXTRA_PKGS}" \
 			--env OSBUILDER_VERSION="${OSBUILDER_VERSION}" \
 			--env OS_VERSION="${OS_VERSION}" \
@@ -481,7 +473,7 @@ setup_rootfs()
 		local unitFile="./etc/systemd/system/tmp.mount"
 		info "Install tmp.mount in ./etc/systemd/system"
 		mkdir -p `dirname "$unitFile"`
-		cp ./usr/share/systemd/tmp.mount "$unitFile" || cat > "$unitFile" << EOF
+		cp ./usr/share/systemd/tmp.mount "$unitFile" || cat > "$unitFile" << EOT
 #  This file is part of systemd.
 #
 #  systemd is free software; you can redistribute it and/or modify it
@@ -504,7 +496,7 @@ What=tmpfs
 Where=/tmp
 Type=tmpfs
 Options=mode=1777,strictatime,nosuid,nodev
-EOF
+EOT
 	fi
 
 	popd  >> /dev/null
@@ -527,12 +519,12 @@ EOF
 	esac
 
 	info "Configure chrony file ${chrony_conf_file}"
-	cat >> "${chrony_conf_file}" <<EOF
+	cat >> "${chrony_conf_file}" <<EOT
 refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0
 # Step the system clock instead of slewing it if the adjustment is larger than
 # one second, at any time
 makestep 1 -1
-EOF
+EOT
 
 	# Comment out ntp sources for chrony to be extra careful
 	# Reference:  https://chrony.tuxfamily.org/doc/3.4/chrony.conf.html
@@ -548,6 +540,11 @@ EOF
 	AGENT_DEST="${AGENT_DIR}/${AGENT_BIN}"
 
 	if [ -z "${AGENT_SOURCE_BIN}" ] ; then
+		if [ "$ARCH" == "ppc64le" ] || [ "$ARCH" == "s390x" ]; then
+			LIBC=gnu
+			echo "WARNING: Forcing LIBC=gnu because $ARCH has no musl Rust target"
+		fi
+		[ "$LIBC" == "musl" ] && bash ${script_dir}/../../../ci/install_musl.sh
 		test -r "${HOME}/.cargo/env" && source "${HOME}/.cargo/env"
 		# rust agent needs ${arch}-unknown-linux-${LIBC}
 		if ! (rustup show | grep -v linux-${LIBC} > /dev/null); then
@@ -558,6 +555,7 @@ EOF
 			bash ${script_dir}/../../../ci/install_rust.sh ${RUST_VERSION}
 		fi
 		test -r "${HOME}/.cargo/env" && source "${HOME}/.cargo/env"
+		[ "$ARCH" == "aarch64" ] && OLD_PATH=$PATH && export PATH=$PATH:/usr/local/musl/bin
 
 		agent_dir="${script_dir}/../../../src/agent/"
 
@@ -565,7 +563,7 @@ EOF
 			info "Set up libseccomp"
 			libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
 			gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
-			${script_dir}/../../../ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
+			bash ${script_dir}/../../../ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
 			echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
 			export LIBSECCOMP_LINK_TYPE=static
 			export LIBSECCOMP_LIB_PATH="${libseccomp_install_dir}/lib"
@@ -579,6 +577,7 @@ EOF
 		make clean
 		make LIBC=${LIBC} INIT=${AGENT_INIT} SECCOMP=${SECCOMP}
 		make install DESTDIR="${ROOTFS_DIR}" LIBC=${LIBC} INIT=${AGENT_INIT}
+		[ "$ARCH" == "aarch64" ] && export PATH=$OLD_PATH && rm -rf /usr/local/musl
 		if [ "${SECCOMP}" == "yes" ]; then
 			rm -rf "${libseccomp_install_dir}" "${gperf_install_dir}"
 		fi

tools/osbuilder/rootfs-builder/template/Dockerfile.template

@@ -14,4 +14,5 @@ FROM ${IMAGE_REGISTRY}/@distro@:@OS_VERSION@
 # RUN commands
 
 # This will install the proper packages to build Kata components
+@INSTALL_MUSL@
 @INSTALL_RUST@

tools/osbuilder/rootfs-builder/ubuntu/Dockerfile-aarch64.in

@@ -0,0 +1,43 @@
+#
+# Copyright (c) 2020 ARM Limited
+#
+# SPDX-License-Identifier: Apache-2.0
+
+ARG IMAGE_REGISTRY=docker.io
+#ubuntu: docker image to be used to create a rootfs
+#@OS_VERSION@: Docker image version to build this dockerfile
+FROM ${IMAGE_REGISTRY}/ubuntu:@OS_VERSION@
+
+# This dockerfile needs to provide all the componets need to build a rootfs
+# Install any package need to create a rootfs (package manager, extra tools)
+
+# Avoid tzdata setup
+ENV DEBIAN_FRONTEND noninteractive
+
+# RUN commands
+RUN apt-get update && apt-get install -y \
+    autoconf \
+    automake \
+    binutils \
+    build-essential \
+    chrony \
+    coreutils \
+    curl \
+    debianutils \
+    debootstrap \
+    g++ \
+    gcc \
+    git \
+    libc6-dev \
+    libstdc++-8-dev \
+    m4 \
+    make \
+    sed \
+    systemd \
+    tar \
+    vim && \
+    apt-get clean && rm -rf /var/lib/apt/lists/
+
+# This will install the proper packages to build Kata components
+@INSTALL_MUSL@
+@INSTALL_RUST@

tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in

@@ -1,29 +1,45 @@
-# Copyright (c) 2018 Yash Jain, 2022 IBM Corp.
+#
+# Copyright (c) 2018  Yash Jain
 #
 # SPDX-License-Identifier: Apache-2.0
 
 ARG IMAGE_REGISTRY=docker.io
+#ubuntu: docker image to be used to create a rootfs
+#@OS_VERSION@: Docker image version to build this dockerfile
 FROM ${IMAGE_REGISTRY}/ubuntu:@OS_VERSION@
-@SET_PROXY@
 
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive \
-    apt-get --no-install-recommends -y install \
+# This dockerfile needs to provide all the componets need to build a rootfs
+# Install any package need to create a rootfs (package manager, extra tools)
+
+# RUN commands
+RUN apt-get update && apt-get --no-install-recommends install -y \
+    apt-utils \
+    autoconf \
+    automake \
+    binutils \
+    build-essential \
     ca-certificates \
+    chrony \
+    coreutils \
     curl \
+    debianutils \
+    debootstrap \
     g++ \
-    $(gcc_arch="@ARCH@" && [ "$(uname -m)" != "$gcc_arch" ] && ( \
-         libc_arch="$gcc_arch" && \
-         [ "$gcc_arch" = aarch64 ] && libc_arch=arm64; \
-         [ "$gcc_arch" = ppc64le ] && gcc_arch=powerpc64le && libc_arch=ppc64el; \
-         [ "$gcc_arch" = x86_64 ] && gcc_arch=x86-64 && libc_arch=amd64; \
-         echo "gcc-$gcc_arch-linux-gnu libc6-dev-$libc_arch-cross")) \
+    gcc \
     git \
+    libc6-dev \
+    libstdc++-8-dev \
+    m4 \
     make \
-    multistrap \
+    musl \
+    musl-dev \
     musl-tools \
-    protobuf-compiler
-# aarch64 requires this name -- link for all
-RUN ln -s /usr/bin/musl-gcc "/usr/bin/$(uname -m)-linux-musl-gcc"
+    protobuf-compiler \
+    sed \
+    systemd \
+    tar \
+    vim \
+    wget
 
+# This will install the proper packages to build Kata components
 @INSTALL_RUST@

tools/osbuilder/rootfs-builder/ubuntu/config.sh

@@ -1,28 +1,34 @@
-# Copyright (c) 2018 Yash Jain, 2022 IBM Corp.
+# This is a configuration file add extra variables to
+#
+# Copyright (c) 2018  Yash Jain
 #
 # SPDX-License-Identifier: Apache-2.0
+# be used by build_rootfs() from rootfs_lib.sh the variables will be
+# loaded just before call the function. For more information see the
+# rootfs-builder/README.md file.
 
-OS_NAME=ubuntu
+OS_VERSION=${OS_VERSION:-20.04}
 # This should be Ubuntu's code name, e.g. "focal" (Focal Fossa) for 20.04
-OS_VERSION=${OS_VERSION:-focal}
-PACKAGES=chrony
-[ "$AGENT_INIT" = no ] && PACKAGES+=" init"
-[ "$SECCOMP" = yes ] && PACKAGES+=" libseccomp2"
-REPO_URL=http://ports.ubuntu.com
+OS_NAME=${OS_NAME:-"focal"}
 
-case "$ARCH" in
-	aarch64) DEB_ARCH=arm64;;
-	ppc64le) DEB_ARCH=ppc64el;;
-	s390x) DEB_ARCH="$ARCH";;
-	x86_64) DEB_ARCH=amd64; REPO_URL=http://archive.ubuntu.com/ubuntu;;
-	*) die "$ARCH not supported"
+# packages to be installed by default
+PACKAGES="systemd coreutils init kmod"
+EXTRA_PKGS+=" chrony"
+
+DEBOOTSTRAP=${PACKAGE_MANAGER:-"debootstrap"}
+
+case $(uname -m) in
+	x86_64) ARCHITECTURE="amd64";;
+	ppc64le) ARCHITECTURE="ppc64el";;
+	aarch64) ARCHITECTURE="arm64";;
+	s390x)	ARCHITECTURE="s390x";;
+	(*) die "$(uname -m) not supported "
 esac
 
-if [ "$(uname -m)" != "$ARCH" ]; then
-	case "$ARCH" in
-		ppc64le) cc_arch=powerpc64le;;
-		x86_64) cc_arch=x86-64;;
-		*) cc_arch="$ARCH"
-	esac
-	export CC="$cc_arch-linux-gnu-gcc"
-fi
+# Init process must be one of {systemd,kata-agent}
+INIT_PROCESS=systemd
+# List of zero or more architectures to exclude from build,
+# as reported by  `uname -m`
+ARCH_EXCLUDE_LIST=()
+
+[ "$SECCOMP" = "yes" ] && PACKAGES+=" libseccomp2" || true

tools/osbuilder/rootfs-builder/ubuntu/rootfs_lib.sh

@@ -1,29 +1,78 @@
-# Copyright (c) 2018 Yash Jain, 2022 IBM Corp.
+# - Arguments
+#
+# Copyright (c) 2018  Yash Jain
 #
 # SPDX-License-Identifier: Apache-2.0
-
+#
+#
+# rootfs_dir=$1
+#
+# - Optional environment variables
+#
+# EXTRA_PKGS: Variable to add extra PKGS provided by the user
+#
+# BIN_AGENT: Name of the Kata-Agent binary
+#
+# REPO_URL: URL to distribution repository ( should be configured in
+#			config.sh file)
+#
+# Any other configuration variable for a specific distro must be added
+# and documented on its own config.sh
+#
+# - Expected result
+#
+# rootfs_dir populated with rootfs pkgs
+# It must provide a binary in /sbin/init
+#
 build_rootfs() {
-	local rootfs_dir=$1
-	local multistrap_conf=multistrap.conf
+	# Mandatory
+	local ROOTFS_DIR=$1
 
-	# For simplicity's sake, use multistrap for foreign and native bootstraps.
-	cat > "$multistrap_conf" << EOF
-[General]
-cleanup=true
-aptsources=Ubuntu
-bootstrap=Ubuntu
+	# Name of the Kata-Agent binary
+	local BIN_AGENT=${BIN_AGENT}
 
-[Ubuntu]
-source=$REPO_URL
-keyring=ubuntu-keyring
-suite=focal
-packages=$PACKAGES $EXTRA_PKGS
-EOF
-	multistrap -a "$DEB_ARCH" -d "$rootfs_dir" -f "$multistrap_conf"
-	rm -rf "$rootfs_dir/var/run"
-	ln -s /run "$rootfs_dir/var/run"
-	cp --remove-destination /etc/resolv.conf "$rootfs_dir/etc"
+	# In case of support EXTRA packages, use it to allow
+	# users to add more packages to the base rootfs
+	local EXTRA_PKGS=${EXTRA_PKGS:-}
 
-	# Reduce image size and memory footprint by removing unnecessary files and directories.
-	rm -rf $rootfs_dir/usr/share/{bash-completion,bug,doc,info,lintian,locale,man,menu,misc,pixmaps,terminfo,zsh}
+	# In case rootfs is created using repositories allow user to modify
+	# the default URL
+	local REPO_URL=${REPO_URL:-YOUR_REPO}
+
+	# PATH where files this script is placed
+	# Use it to refer to files in the same directory
+	# Example: ${CONFIG_DIR}/foo
+	local CONFIG_DIR=${CONFIG_DIR}
+
+
+	# Populate ROOTFS_DIR
+	# Must provide /sbin/init and /bin/${BIN_AGENT}
+	DEBOOTSTRAP="debootstrap"
+	check_root
+	mkdir -p "${ROOTFS_DIR}"
+	if [ -n "${PKG_MANAGER}"  ]; then
+		info "debootstrap path provided by user: ${PKG_MANAGER}"
+	elif check_program $DEBOOTSTRAP ; then
+		PKG_MANAGER=$DEBOOTSTRAP
+	else
+		die "$DEBOOTSTRAP is not installed"
+	fi
+	# trim whitespace
+	PACKAGES=$(echo $PACKAGES |xargs )
+	# add comma as debootstrap needs , separated package names.
+	# Don't change $PACKAGES in config.sh to include ','
+	# This is done to maintain consistency
+	PACKAGES=$(echo $PACKAGES | sed  -e 's/ /,/g' )
+
+	${PKG_MANAGER} --variant=minbase \
+		--arch=${ARCHITECTURE}\
+		--include="$PACKAGES" \
+		${OS_NAME} \
+		${ROOTFS_DIR}
+
+	[ -n "${EXTRA_PKGS}" ] && chroot $ROOTFS_DIR apt-get install -y ${EXTRA_PKGS}
+
+    # Reduce image size and memory footprint
+    # removing not needed files and directories.
+    chroot $ROOTFS_DIR rm -rf /usr/share/{bash-completion,bug,doc,info,lintian,locale,man,menu,misc,pixmaps,terminfo,zoneinfo,zsh}
 }

tools/osbuilder/scripts/lib.sh

@@ -7,6 +7,7 @@
 set -e
 
 KATA_REPO=${KATA_REPO:-github.com/kata-containers/kata-containers}
+MUSL_VERSION=${MUSL_VERSION:-"null"}
 # Give preference to variable set by CI
 yq_file="${script_dir}/../../../ci/install_yq.sh"
 kata_versions_file="${script_dir}/../../../versions.yaml"
@@ -167,7 +168,7 @@ create_summary_file()
 	local -r agentdir="${script_dir}/../../../"
 	local -r agent_version=$(cat ${agentdir}/VERSION)
 
-	cat >"$file"<<-EOF
+	cat >"$file"<<-EOT
 	---
 	osbuilder:
 	  url: "${osbuilder_url}"
@@ -189,7 +190,7 @@ ${extra}
 	  name: "${AGENT_BIN}"
 	  version: "${agent_version}"
 	  agent-is-init-daemon: "${AGENT_INIT}"
-EOF
+EOT
 
 	local rootfs_file="${file_dir}/$(basename "${file}")"
 	info "Created summary file '${rootfs_file}' inside rootfs"
@@ -203,27 +204,107 @@ generate_dockerfile()
 	dir="$1"
 	[ -d "${dir}" ] || die "${dir}: not a directory"
 
-	local rustarch="$ARCH"
-	[ "$ARCH" = ppc64le ] && rustarch=powerpc64le
+	local architecture=$(uname -m)
+	local rustarch=${architecture}
+	local muslarch=${architecture}
+	local libc=musl
+	case "$(uname -m)" in
+		"ppc64le")
+			rustarch=powerpc64le
+			muslarch=powerpc64
+			libc=gnu
+			;;
+		"s390x")
+			libc=gnu
+			;;
+
+		*)
+			;;
+	esac
 
 	[ -n "${http_proxy:-}" ] && readonly set_proxy="RUN sed -i '$ a proxy="${http_proxy:-}"' /etc/dnf/dnf.conf /etc/yum.conf; true"
 
 	# Rust agent
+	# rust installer should set path apropiately, just in case
+	# install musl for compiling rust-agent
+	local musl_source_url="https://git.zv.io/toolchains/musl-cross-make.git"
+	local musl_source_dir="musl-cross-make"
+	install_musl=
+	if [ "${muslarch}" == "aarch64" ]; then
+		local musl_tar="${muslarch}-linux-musl-native.tgz"
+		local musl_dir="${muslarch}-linux-musl-native"
+		local aarch64_musl_target="aarch64-linux-musl"
+		install_musl="
+RUN cd /tmp; \
+	mkdir -p /usr/local/musl/; \
+	if curl -sLO --fail https://musl.cc/${musl_tar}; then \
+		tar -zxf ${musl_tar}; \
+		cp -r ${musl_dir}/* /usr/local/musl/; \
+	else \
+		git clone ${musl_source_url}; \
+		TARGET=${aarch64_musl_target} make -j$(nproc) -C ${musl_source_dir} install; \
+		cp -r ${musl_source_dir}/output/* /usr/local/musl/; \
+		cp /usr/local/musl/bin/aarch64-linux-musl-g++ /usr/local/musl/bin/g++; \
+	fi
+ENV PATH=\$PATH:/usr/local/musl/bin
+RUN ln -sf /usr/local/musl/bin/g++ /usr/bin/g++
+"
+	else
+		local musl_tar="musl-${MUSL_VERSION}.tar.gz"
+		local musl_dir="musl-${MUSL_VERSION}"
+		install_musl="
+RUN pushd /root; \
+    curl -sLO https://www.musl-libc.org/releases/${musl_tar}; tar -zxf ${musl_tar}; \
+	cd ${musl_dir}; \
+	sed -i \"s/^ARCH = .*/ARCH = ${muslarch}/g\" dist/config.mak; \
+	./configure > /dev/null 2>\&1; \
+	make > /dev/null 2>\&1; \
+	make install > /dev/null 2>\&1; \
+	echo \"/usr/local/musl/lib\" > /etc/ld-musl-${muslarch}.path; \
+	popd
+ENV PATH=\$PATH:/usr/local/musl/bin
+"
+	fi
+
 	readonly install_rust="
-ENV http_proxy=${http_proxy:-}
-ENV https_proxy=${http_proxy:-}
-RUN curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSLf | \
-    sh -s -- -y --default-toolchain ${RUST_VERSION} -t ${rustarch}-unknown-linux-${LIBC}
-RUN . /root/.cargo/env; cargo install cargo-when
+RUN curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSLf --output /tmp/rust-init; \
+    chmod a+x /tmp/rust-init; \
+	export http_proxy=${http_proxy:-}; \
+	export https_proxy=${http_proxy:-}; \
+	/tmp/rust-init -y --default-toolchain ${RUST_VERSION}
+RUN . /root/.cargo/env; \
+    export http_proxy=${http_proxy:-}; \
+	export https_proxy=${http_proxy:-}; \
+	cargo install cargo-when; \
+	rustup target install ${rustarch}-unknown-linux-${libc}
+RUN ln -sf /usr/bin/g++ /bin/musl-g++
 "
 	pushd "${dir}"
+	dockerfile_template="Dockerfile.in"
+	dockerfile_arch_template="Dockerfile-${architecture}.in"
+	# if arch-specific docker file exists, swap the univesal one with it.
+        if [ -f "${dockerfile_arch_template}" ]; then
+                dockerfile_template="${dockerfile_arch_template}"
+        else
+                [ -f "${dockerfile_template}" ] || die "${dockerfile_template}: file not found"
+        fi
 
-	sed \
-		-e "s#@OS_VERSION@#${OS_VERSION:-}#g" \
-		-e "s#@ARCH@#$ARCH#g" \
-		-e "s#@INSTALL_RUST@#${install_rust//$'\n'/\\n}#g" \
-		-e "s#@SET_PROXY@#${set_proxy:-}#g" \
-		Dockerfile.in > Dockerfile
+	# ppc64le and s390x have no musl target
+	if [ "${architecture}" == "ppc64le" ] || [ "${architecture}" == "s390x" ]; then
+		sed \
+			-e "s|@OS_VERSION@|${OS_VERSION:-}|g" \
+			-e "s|@INSTALL_MUSL@||g" \
+			-e "s|@INSTALL_RUST@|${install_rust//$'\n'/\\n}|g" \
+			-e "s|@SET_PROXY@|${set_proxy:-}|g" \
+			"${dockerfile_template}" > Dockerfile
+	else
+		sed \
+			-e "s|@OS_VERSION@|${OS_VERSION:-}|g" \
+			-e "s|@INSTALL_MUSL@|${install_musl//$'\n'/\\n}|g" \
+			-e "s|@INSTALL_RUST@|${install_rust//$'\n'/\\n}|g" \
+			-e "s|@SET_PROXY@|${set_proxy:-}|g" \
+			"${dockerfile_template}" > Dockerfile
+	fi
 	popd
 }
 
@@ -264,6 +345,17 @@ detect_rust_version()
 	[ -n "$RUST_VERSION" ]
 }
 
+detect_musl_version()
+{
+	info "Detecting musl version"
+    local yq_path="externals.musl.version"
+
+	info "Get musl version from ${kata_versions_file}"
+	MUSL_VERSION="$(get_package_version_from_kata_yaml "$yq_path")"
+
+	[ -n "$MUSL_VERSION" ]
+}
+
 before_starting_container() {
 	return 0
 }

tools/osbuilder/tests/test_images.sh

@@ -53,7 +53,7 @@ source "${project_dir}/scripts/lib.sh"
 
 usage()
 {
-	cat <<EOF
+	cat <<EOT
 Usage: $script_name [options] [command | <distro>]
 
 Options:
@@ -72,7 +72,7 @@ Otherwise, tests are run on all distros.
 $(basename ${test_config}) includes a list of distros to exclude from testing,
 depending on the detected test environment. However, when a <distro> is specified,
 distro exclusion based on $(basename ${test_config}) is not enforced.
-EOF
+EOT
 }
 
 # Add an entry to the specified stats file
@@ -640,6 +640,8 @@ test_dracut()
 		die "Could not detect the required Go version for AGENT_VERSION='${AGENT_VERSION:-master}'."
 	detect_rust_version ||
 		die "Could not detect the required rust version for AGENT_VERSION='${AGENT_VERSION:-master}'."
+	detect_musl_version ||
+		die "Could not detect the required musl version for AGENT_VERSION='${AGENT_VERSION:-master}'."
 
 	generate_dockerfile ${dracut_dir}
 	info "Creating container for dracut"

tools/packaging/guest-image/build_image.sh

@@ -59,7 +59,7 @@ build_image() {
 
 usage() {
 	return_code=${1:-0}
-	cat <<EOF
+	cat <<EOT
 Create image and initrd in a tarball for kata containers.
 Use it to build an image to distribute kata.
 
@@ -70,7 +70,7 @@ Options:
  --imagetype=${image_type}
  --prefix=${prefix}
  --destdir=${destdir}
-EOF
+EOT
 
 	exit "${return_code}"
 }

tools/packaging/kata-deploy/README.md

@@ -27,7 +27,7 @@ The stable image refers to the last stable releases content.
 > **Note:** if you use a tagged version of the repo, the stable image does match that version.
 > For instance, if you use the 2.2.1 tagged version of the kata-deploy.yaml file, then the version 2.2.1 of the kata runtime will be deployed.
 
-```bash
+```sh
 $ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
 $ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml
 ```
@@ -36,15 +36,12 @@ $ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-contai
 
 ```sh
 $ GO111MODULE=auto go get github.com/kata-containers/kata-containers
-```
-
-```bash
 $ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/packaging/kata-deploy
 $ kubectl apply -k kata-deploy/overlays/k3s
 ```
 
 #### Ensure kata-deploy is ready
-```bash
+```sh
 kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
 ```
 
@@ -55,7 +52,7 @@ the `Pod` specification. The `runtimeClass` examples provided define a node sele
 which will ensure the workload is only scheduled on a node that has Kata Containers installed
 
 `runtimeClass` is a built-in type in Kubernetes. To apply each Kata Containers `runtimeClass`:
-```bash
+```sh
   $ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
 ```
 
@@ -88,25 +85,25 @@ spec:
 
 To run an example with `kata-clh`:
 
-```bash
+```sh
 $ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/examples/test-deploy-kata-clh.yaml
 ```
 
 To run an example with `kata-fc`:
 
-```bash
+```sh
 $ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/examples/test-deploy-kata-fc.yaml
 ```
 
 To run an example with `kata-qemu`:
 
-```bash
+```sh
 $ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/examples/test-deploy-kata-qemu.yaml
 ```
 
 The following removes the test pods:
 
-```bash
+```sh
 $ kubectl delete -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/examples/test-deploy-kata-clh.yaml
 $ kubectl delete -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/examples/test-deploy-kata-fc.yaml
 $ kubectl delete -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/examples/test-deploy-kata-qemu.yaml
@@ -139,13 +136,13 @@ $ kubectl delete -f https://raw.githubusercontent.com/kata-containers/kata-conta
 
 #### Removing the stable image
 
-```bash
+```sh
 $ kubectl delete -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml
 $ kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
 ```
 
 After ensuring kata-deploy has been deleted, cleanup the cluster:
-```bash
+```sh
 $ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml
 ```
 
@@ -153,7 +150,7 @@ The cleanup daemon-set will run a single time, cleaning up the node-label, which
 This process should take, at most, 5 minutes.
 
 After that, let's delete the cleanup daemon-set, the added RBAC and runtime classes:
-```bash
+```sh
 $ kubectl delete -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml
 $ kubectl delete -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
 $ kubectl delete -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml

tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml

@@ -1,46 +0,0 @@
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kubelet-kata-cleanup
-  namespace: kube-system
-spec:
-  selector:
-      matchLabels:
-        name: kubelet-kata-cleanup
-  template:
-    metadata:
-        labels:
-          name: kubelet-kata-cleanup
-    spec:
-      serviceAccountName: kata-label-node
-      nodeSelector:
-          katacontainers.io/kata-runtime: cleanup
-      containers:
-      - name: kube-kata-cleanup
-        image: quay.io/kata-containers/kata-deploy:stable
-        imagePullPolicy: Always
-        command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh reset" ]
-        env:
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        securityContext:
-          privileged: false
-        volumeMounts:
-        - name: dbus
-          mountPath: /var/run/dbus
-        - name: systemd
-          mountPath: /run/systemd
-      volumes:
-        - name: dbus
-          hostPath:
-            path: /var/run/dbus
-        - name: systemd
-          hostPath:
-            path: /run/systemd
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 1
-    type: RollingUpdate

tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml

@@ -18,7 +18,7 @@ spec:
           katacontainers.io/kata-runtime: cleanup
       containers:
       - name: kube-kata-cleanup
-        image: quay.io/kata-containers/kata-deploy:latest
+        image: quay.io/kata-containers/kata-deploy:2.4.0
         imagePullPolicy: Always
         command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh reset" ]
         env:

tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml

@@ -1,69 +0,0 @@
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kata-deploy
-  namespace: kube-system
-spec:
-  selector:
-      matchLabels:
-        name: kata-deploy
-  template:
-    metadata:
-        labels:
-          name: kata-deploy
-    spec:
-      serviceAccountName: kata-label-node
-      containers:
-      - name: kube-kata
-        image: quay.io/kata-containers/kata-deploy:stable
-        imagePullPolicy: Always
-        lifecycle:
-          preStop:
-            exec:
-              command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"]
-        command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh install" ]
-        env:
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        securityContext:
-          privileged: false
-        volumeMounts:
-        - name: crio-conf
-          mountPath: /etc/crio/
-        - name: containerd-conf
-          mountPath: /etc/containerd/
-        - name: kata-artifacts
-          mountPath: /opt/kata/
-        - name: dbus
-          mountPath: /var/run/dbus
-        - name: systemd
-          mountPath: /run/systemd
-        - name: local-bin
-          mountPath: /usr/local/bin/
-      volumes:
-        - name: crio-conf
-          hostPath:
-            path: /etc/crio/
-        - name: containerd-conf
-          hostPath:
-            path: /etc/containerd/
-        - name: kata-artifacts
-          hostPath:
-            path: /opt/kata/
-            type: DirectoryOrCreate
-        - name: dbus
-          hostPath:
-            path: /var/run/dbus
-        - name: systemd
-          hostPath:
-            path: /run/systemd
-        - name: local-bin
-          hostPath:
-            path: /usr/local/bin/
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 1
-    type: RollingUpdate

tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml

@@ -16,7 +16,7 @@ spec:
       serviceAccountName: kata-label-node
       containers:
       - name: kube-kata
-        image: quay.io/kata-containers/kata-deploy:latest
+        image: quay.io/kata-containers/kata-deploy:2.4.0
         imagePullPolicy: Always
         lifecycle:
           preStop:

tools/packaging/kata-deploy/local-build/.gitignore

@@ -1 +0,0 @@
-build/

tools/packaging/kata-deploy/local-build/Makefile

@@ -15,11 +15,8 @@ endef
 
 kata-tarball: | all-parallel merge-builds
 
-$(MK_DIR)/dockerbuild/install_yq.sh:
-	$(MK_DIR)/kata-deploy-copy-yq-installer.sh
-
-all-parallel: $(MK_DIR)/dockerbuild/install_yq.sh
-	${MAKE} -f $(MK_PATH) all -j$$(( $$(nproc) - 1  )) V=
+all-parallel:
+	${MAKE} -f $(MK_PATH) all -j$$(( $$(nproc) - 1  )) NO_TTY="true" V=
 
 all: cloud-hypervisor-tarball \
 	firecracker-tarball \
@@ -29,7 +26,7 @@ all: cloud-hypervisor-tarball \
 	rootfs-initrd-tarball \
 	shim-v2-tarball
 
-%-tarball-build: $(MK_DIR)/dockerbuild/install_yq.sh
+%-tarball-build:
 	$(call BUILD,$*)
 
 cloud-hypervisor-tarball:

tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh

@@ -16,17 +16,25 @@ kata_deploy_create="${script_dir}/kata-deploy-binaries.sh"
 uid=$(id -u ${USER})
 gid=$(id -g ${USER})
 
+TTY_OPT="-i"
+NO_TTY="${NO_TTY:-false}"
+[ -t 1 ] &&  [ "${NO_TTY}"  == "false" ] && TTY_OPT="-it"
+
 if [ "${script_dir}" != "${PWD}" ]; then
 	ln -sf "${script_dir}/build" "${PWD}/build"
 fi
 
+install_yq_script_path="${script_dir}/../../../../ci/install_yq.sh"
+
+cp "${install_yq_script_path}" "${script_dir}/dockerbuild/install_yq.sh"
+
 docker build -q -t build-kata-deploy \
 	--build-arg IMG_USER="${USER}" \
 	--build-arg UID=${uid} \
 	--build-arg GID=${gid} \
 	"${script_dir}/dockerbuild/"
 
-docker run \
+docker run ${TTY_OPT} \
 	-v /var/run/docker.sock:/var/run/docker.sock \
 	--user ${uid}:${gid} \
 	--env USER=${USER} -v "${kata_dir}:${kata_dir}" \

tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh

@@ -8,7 +8,6 @@
 set -o errexit
 set -o nounset
 set -o pipefail
-set -o errtrace
 
 readonly project="kata-containers"
 
@@ -51,7 +50,7 @@ error() {
 
 usage() {
 	return_code=${1:-0}
-	cat <<EOF
+	cat <<EOT
 This script is used as part of the ${project} release process.
 It is used to create a tarball with static binaries.
 
@@ -65,7 +64,6 @@ version: The kata version that will be use to create the tarball
 options:
 
 -h|--help      	      : Show this help
--s             	      : Silent mode (produce output in case of failure only)
 --build=<asset>       :
 	all
 	cloud-hypervisor
@@ -76,7 +74,7 @@ options:
 	rootfs-image
 	rootfs-initrd
 	shim-v2
-EOF
+EOT
 
 	exit "${return_code}"
 }
@@ -197,18 +195,6 @@ handle_build() {
 	tar tvf "${tarball_name}"
 }
 
-silent_mode_error_trap() {
-	local stdout="$1"
-	local stderr="$2"
-	local t="$3"
-	local log_file="$4"
-	exec 1>&${stdout}
-	exec 2>&${stderr}
-	error "Failed to build: $t, logs:"
-	cat "${log_file}"
-	exit 1
-}
-
 main() {
 	local build_targets
 	local silent
@@ -261,15 +247,11 @@ main() {
 		(
 			cd "${builddir}"
 			if [ "${silent}" == true ]; then
-				local stdout
-				local stderr
-				# Save stdout and stderr, to be restored
-				# by silent_mode_error_trap() in case of
-				# build failure.
-				exec {stdout}>&1
-				exec {stderr}>&2
-				trap "silent_mode_error_trap $stdout $stderr $t \"$log_file\"" ERR
-				handle_build "${t}" &>"$log_file"
+				if ! handle_build "${t}" &>"$log_file"; then
+					error "Failed to build: $t, logs:"
+					cat "${log_file}"
+					exit 1
+				fi
 			else
 				handle_build "${t}"
 			fi

tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018-2021 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-set -o errexit
-set -o nounset
-set -o pipefail
-set -o errtrace
-
-script_dir=$(dirname "$(readlink -f "$0")")
-install_yq_script_path="${script_dir}/../../../../ci/install_yq.sh"
-
-cp "${install_yq_script_path}" "${script_dir}/dockerbuild/install_yq.sh"

tools/packaging/kata-deploy/scripts/kata-deploy.sh

@@ -94,10 +94,10 @@ function configure_different_shims_base() {
 			fi
 		fi
 
-		cat << EOF | tee "$shim_file"
+		cat << EOT | tee "$shim_file"
 #!/usr/bin/env bash
 KATA_CONF_FILE=/opt/kata/share/defaults/kata-containers/configuration-${shim}.toml /opt/kata/bin/containerd-shim-kata-v2 "\$@"
-EOF
+EOT
 		chmod +x "$shim_file"
 
 		if [ "${shim}" == "${default_shim}" ]; then
@@ -132,7 +132,7 @@ function configure_crio_runtime() {
 	local kata_path="/usr/local/bin/containerd-shim-${runtime}-v2"
 	local kata_conf="crio.runtime.runtimes.${runtime}"
 
-	cat <<EOF | tee -a "$crio_drop_in_conf_file"
+	cat <<EOT | tee -a "$crio_drop_in_conf_file"
 
 # Path to the Kata Containers runtime binary that uses the $1
 [$kata_conf]
@@ -140,7 +140,7 @@ function configure_crio_runtime() {
 	runtime_type = "vm"
 	runtime_root = "/run/vc"
 	privileged_without_host_devices = true
-EOF
+EOT
 }
 
 function configure_crio() {
@@ -178,22 +178,22 @@ function configure_containerd_runtime() {
 		echo "Configuration exists for $runtime_table, overwriting"
 		sed -i "/\[$runtime_table\]/,+1s#runtime_type.*#runtime_type = \"${runtime_type}\"#" $containerd_conf_file
 	else
-		cat <<EOF | tee -a "$containerd_conf_file"
+		cat <<EOT | tee -a "$containerd_conf_file"
 [$runtime_table]
   runtime_type = "${runtime_type}"
   privileged_without_host_devices = true
   pod_annotations = ["io.katacontainers.*"]
-EOF
+EOT
 	fi
 
 	if grep -q "\[$options_table\]" $containerd_conf_file; then
 		echo "Configuration exists for $options_table, overwriting"
 		sed -i "/\[$options_table\]/,+1s#ConfigPath.*#ConfigPath = \"${config_path}\"#" $containerd_conf_file
 	else
-		cat <<EOF | tee -a "$containerd_conf_file"
+		cat <<EOT | tee -a "$containerd_conf_file"
   [$options_table]
     ConfigPath = "${config_path}"
-EOF
+EOT
 	fi
 }
 

tools/packaging/kernel/build-kernel.sh

@@ -65,7 +65,7 @@ source "${packaging_scripts_dir}/lib.sh"
 
 usage() {
 	exit_code="$1"
-	cat <<EOF
+	cat <<EOT
 Overview:
 
 	Build a kernel for Kata Containers
@@ -99,7 +99,7 @@ Options:
 	-t <hypervisor>	: Hypervisor_target.
 	-v <version>	: Kernel version to use if kernel path not provided.
 	-x <type>	: Confidential guest protection type, such as sev and tdx
-EOF
+EOT
 	exit "$exit_code"
 }
 

tools/packaging/release/publish-kata-image.sh

@@ -28,7 +28,7 @@ die() {
 
 usage() {
 	return_code=${1:-0}
-	cat <<EOF
+	cat <<EOT
 Usage:
 
 ${script_name} [options]  <version>
@@ -41,7 +41,7 @@ options:
 
 -h      : show this help
 -p      : push image to github
-EOF
+EOT
 
 	exit "${return_code}"
 }

tools/packaging/release/release-notes.sh

@@ -26,7 +26,7 @@ trap exit_handler EXIT
 
 usage() {
 	return_code=${1:-}
-	cat <<EOF
+	cat <<EOT
 Usage ${script_name} <previous-release> <new_release>
 
 Args:
@@ -37,7 +37,7 @@ new-release:      new release version that will have the
 Example:
 ./${script_name} 1.2.0 1.2.1 > notes.md
 
-EOF
+EOT
 	exit "${return_code}"
 }
 
@@ -81,25 +81,25 @@ changes() {
 }
 
 print_release_notes() {
-	cat <<EOF
+	cat <<EOT
 # Release ${runtime_version}
 
-EOF
+EOT
 
 	for repo in "${repos[@]}"; do
 		git clone -q "https://github.com/${project}/${repo}.git" "${tmp_dir}/${repo}"
 		pushd "${tmp_dir}/${repo}" >>/dev/null
 
-		cat <<EOF
+		cat <<EOT
 ## ${repo} Changes
 $(changes)
 
-EOF
+EOT
 		popd >>/dev/null
 		rm -rf "${tmp_dir}/${repo}"
 	done
 
-	cat <<EOF
+	cat <<EOT
 
 ## Compatibility with CRI-O
 Kata Containers ${runtime_version} is compatible with CRI-O ${crio_version}
@@ -157,7 +157,7 @@ More information [Limitations][limitations]
 [custom-agent-doc]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#build-a-custom-kata-agent---optional
 [limitations]: https://github.com/kata-containers/kata-containers/blob/${new_release}/docs/Limitations.md
 [installation]: https://github.com/kata-containers/kata-containers/blob/${new_release}/docs/install
-EOF
+EOT
 }
 
 main() {

tools/packaging/release/tag_repos.sh

@@ -27,16 +27,16 @@ source "${script_dir}/../scripts/lib.sh"
 
 function usage() {
 
-	cat <<EOF
+	cat <<EOT
 Usage: ${script_name} [options] <args>
 This script creates a new release for ${PROJECT}.
 It tags and create release for:
-EOF
+EOT
 	for r in "${repos[@]}"; do
 		echo "  - ${r}"
 	done
 
-	cat <<EOF
+	cat <<EOT
 
 Args:
 status : Get Current ${PROJECT} tags status
@@ -48,7 +48,7 @@ Options:
 -h         : Show this help
 -p         : push tags
 
-EOF
+EOT
 
 }
 

tools/packaging/release/update-repository-version.sh

@@ -219,12 +219,12 @@ bump_repo() {
 
 	info "Creating PR message"
 	notes_file=notes.md
-	cat <<EOF >"${notes_file}"
+	cat <<EOT >"${notes_file}"
 # Kata Containers ${new_version}
 
 $(get_changes "$current_version")
 
-EOF
+EOT
 	cat "${notes_file}"
 
 	if (echo "${current_version}" | grep "alpha") && (echo "${new_version}" | grep -v "alpha");then
@@ -278,7 +278,7 @@ EOF
 
 usage() {
 	exit_code="$1"
-	cat <<EOF
+	cat <<EOT
 Usage:
 	${script_name} [options] <args>
 Args:
@@ -289,7 +289,7 @@ Example:
 Options
 	-h        : Show this help
 	-p        : create a PR
-EOF
+EOT
 	exit "$exit_code"
 }
 

tools/packaging/scripts/apply_patches.sh

@@ -12,7 +12,7 @@ script_dir="$(realpath $(dirname $0))"
 patches_dir="$1"
 
 if [ -z "$patches_dir" ]; then
-	cat <<-EOF
+	cat <<-EOT
 	Apply patches to the sources at the current directory.
 
 	Patches are expected to be named in the standard git-format-patch(1) format where
@@ -30,7 +30,7 @@ if [ -z "$patches_dir" ]; then
 	    $0 PATCHES_DIR
 	Where:
 	    PATCHES_DIR is the directory containing the patches
-	EOF
+	EOT
 	exit 1
 fi
 

tools/packaging/scripts/configure-hypervisor.sh

@@ -72,7 +72,7 @@ die() {
 
 # Display usage to stdout.
 usage() {
-	cat <<EOF
+	cat <<EOT
 Overview:
 
 	Display configure options required to build the specified
@@ -94,7 +94,7 @@ Example:
 
 	$ $script_name qemu
 
-EOF
+EOT
 }
 
 show_tags_header() {
@@ -102,10 +102,10 @@ show_tags_header() {
 	local key
 	local value
 
-	cat <<EOF
+	cat <<EOT
 # Recognised option tags:
 #
-EOF
+EOT
 
 	# sort the tags
 	keys=${!recognised_tags[@]}

tools/packaging/scripts/gen_versions_txt.sh

@@ -55,7 +55,7 @@ gen_version_file() {
 	# see https://github.com/semver/semver/issues/145
 	kata_version=$(get_kata_version)
 	kata_version=${kata_version/-/\~}
-	cat > "$versions_txt" <<EOF
+	cat > "$versions_txt" <<EOT
 # This is a generated file from ${script_name}
 
 kata_version=${kata_version}
@@ -70,7 +70,7 @@ kernel_version=${kernel_version}
 
 # Golang
 go_version=${golang_version}
-EOF
+EOT
 }
 
 die() {
@@ -85,7 +85,7 @@ die() {
 
 usage() {
 	exit_code=$"${1:-0}"
-	cat <<EOF
+	cat <<EOT
 Usage:
 ${script_name} [--compare | -h | --help] <kata-branch>
 
@@ -98,7 +98,7 @@ Options:
 --compare         Only compare the kata version at branch <kata-branch> with the
                   one in ${versions_txt} and leave the file untouched.
 --head            Use <kata-branch>'s head to generate the versions file.
-EOF
+EOT
 	exit "${exit_code}"
 }
 

tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh

@@ -22,8 +22,6 @@ source "${script_dir}/../../scripts/lib.sh"
 
 cloud_hypervisor_repo="${cloud_hypervisor_repo:-}"
 cloud_hypervisor_version="${cloud_hypervisor_version:-}"
-cloud_hypervisor_pr="${cloud_hypervisor_pr:-}"
-cloud_hypervisor_pull_ref_branch="${cloud_hypervisor_pull_ref_branch:-main}"
 
 if [ -z "$cloud_hypervisor_repo" ]; then
        info "Get cloud_hypervisor information from runtime versions.yaml"
@@ -33,13 +31,8 @@ if [ -z "$cloud_hypervisor_repo" ]; then
 fi
 [ -n "$cloud_hypervisor_repo" ] || die "failed to get cloud_hypervisor repo"
 
-if [ -n "$cloud_hypervisor_pr" ]; then
-	force_build_from_source=true
-	cloud_hypervisor_version="PR $cloud_hypervisor_pr"
-else
-	[ -n "$cloud_hypervisor_version" ] || cloud_hypervisor_version=$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version" "${kata_version}")
-	[ -n "$cloud_hypervisor_version" ] || die "failed to get cloud_hypervisor version"
-fi
+[ -n "$cloud_hypervisor_version" ] || cloud_hypervisor_version=$(get_from_kata_deps "assets.hypervisor.cloud_hypervisor.version" "${kata_version}")
+[ -n "$cloud_hypervisor_version" ] || die "failed to get cloud_hypervisor version"
 
 pull_clh_released_binary() {
     info "Download cloud-hypervisor version: ${cloud_hypervisor_version}"
@@ -57,19 +50,8 @@ build_clh_from_source() {
     repo_dir="${repo_dir//.git}"
     [ -d "${repo_dir}" ] || git clone "${cloud_hypervisor_repo}"
     pushd "${repo_dir}"
-
-    if [ -n "${cloud_hypervisor_pr}" ]; then
-	    local pr_branch="PR_${cloud_hypervisor_pr}"
-	    git fetch origin "pull/${cloud_hypervisor_pr}/head:${pr_branch}" || return 1
-	    git checkout "${pr_branch}"
-	    git rebase "origin/${cloud_hypervisor_pull_ref_branch}"
-
-	    git log --oneline main~1..HEAD
-    else
-	    git fetch || true
-	    git checkout "${cloud_hypervisor_version}"
-    fi 
-
+    git fetch || true
+    git checkout "${cloud_hypervisor_version}"
     if [ -n "${features}" ]; then
         info "Build cloud-hypervisor enabling the following features: ${features}"
         ./scripts/dev_cli.sh build --release --libc musl --features "${features}"

tools/packaging/static-build/shim-v2/install_go.sh

@@ -31,7 +31,7 @@ info() {
 
 usage(){
 	exit_code="$1"
-	cat <<EOF
+	cat <<EOT
 Usage:
 
 ${script_name} [options]
@@ -43,7 +43,7 @@ Options
 -d <path> : destination path, path where go will be installed.
 -f        : enable force install, remove existent go pkg before installation.
 -h        : display this help.
-EOF
+EOT
 
 	exit "$exit_code"
 }

utils.mk

@@ -113,8 +113,7 @@ endef
 BUILD_TYPE = release
 
 ##VAR ARCH=arch target to build (format: uname -m)
-HOST_ARCH = $(shell uname -m)
-ARCH ?= $(HOST_ARCH)
+ARCH = $(shell uname -m)
 ##VAR LIBC=musl|gnu
 LIBC ?= musl
 ifneq ($(LIBC),musl)
@@ -143,14 +142,6 @@ ifeq ($(ARCH), aarch64)
     $(warning "WARNING: aarch64-musl needs extra symbols from libgcc")
 endif
 
-ifneq ($(HOST_ARCH),$(ARCH))
-    ifeq ($(CC),)
-         CC = gcc
-         $(warning "WARNING: A foreign ARCH was passed, but no CC alternative. Using gcc.")
-    endif
-    override EXTRA_RUSTFLAGS += -C linker=$(CC)
-endif
-
 TRIPLE = $(ARCH)-unknown-linux-$(LIBC)
 
 CWD := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))

utils/kata-manager.sh

@@ -43,7 +43,7 @@ readonly link_dir=${link_dir:-/usr/bin}
 
 readonly tmpdir=$(mktemp -d)
 
-readonly warnings=$(cat <<EOF
+readonly warnings=$(cat <<EOT
 WARNINGS:
 
 - Use distro-packages where possible
@@ -63,7 +63,7 @@ WARNINGS:
   and $containerd_project from binary release packages. These versions may
   not have been tested with your distribution version.
 
-EOF
+EOT
 )
 
 die()
@@ -199,7 +199,7 @@ github_download_release()
 
 usage()
 {
-	cat <<EOF
+	cat <<EOT
 Usage: $script_name [options] [<kata-version> [<containerd-version>]]
 
 Description: Install $kata_project [1] (and optionally $containerd_project [2])
@@ -235,7 +235,7 @@ Advice:
 
   $ kata-runtime check --only-list-releases
 
-EOF
+EOT
 }
 
 # Determine if the system only supports cgroups v2.
@@ -447,6 +447,7 @@ configure_containerd()
 			"$(date -Iseconds)" |\
 			tee -a "$containerd_service_name"
 
+
 		sudo cp "${containerd_service_name}" "${dest}"
 		sudo systemctl daemon-reload
 
@@ -480,7 +481,7 @@ configure_containerd()
 		"$script_name")
 
 	sudo grep -q "$kata_runtime_type" "$cfg" || {
-		cat <<-EOF | sudo tee -a "$cfg"
+		cat <<-EOT | sudo tee -a "$cfg"
 		# $comment_text
 		[plugins]
 		  [plugins."io.containerd.grpc.v1.cri"]
@@ -489,7 +490,7 @@ configure_containerd()
 		      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
 		        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.${kata_runtime_name}]
 		          runtime_type = "${kata_runtime_type}"
-		EOF
+		EOT
 
 		modified="true"
 	}
@@ -501,11 +502,11 @@ configure_containerd()
 			grep -E "^\s*\<level\>\s*=\s*.*\<debug\>" || true)
 
 		[ -n "$debug_enabled" ] || {
-			cat <<-EOF | sudo tee -a "$cfg"
+			cat <<-EOT | sudo tee -a "$cfg"
 			# $comment_text
 			[debug]
 				level = "debug"
-			EOF
+			EOT
 		}
 
 		modified="true"

versions.yaml

@@ -83,7 +83,7 @@ assets:
       uscan-url: >-
         https://github.com/firecracker-microvm/firecracker/tags
         .*/v?(\d\S+)\.tar\.gz
-      version: "v0.23.4"
+      version: "v0.23.1"
 
     qemu:
       description: "VMM that uses KVM"
@@ -233,6 +233,19 @@ externals:
       .*/v?(\d\S+)\.tar\.gz
     version: "v1.0.1"
 
+  musl:
+    description: |
+      The musl library is used to build the rust agent.
+    url: "https://www.musl-libc.org/"
+    uscan-url: >-
+      https://www.musl-libc.org/releases/
+      musl-([\d\.]+)\.tar\.gz
+    version: "1.1.23"
+    meta:
+      description: |
+        'newest-version' is the latest version known to work.
+      newest-version: "1.1.23"
+
   nydus:
     description: "Nydus image acceleration service"
     url: "https://github.com/dragonflyoss/image-service"