Merge pull request #7231 from wainersm/measured_rootfs-improvements

Build for measured rootfs improvements

src/runtime-rs/Makefile

@@ -452,12 +452,15 @@ optimize: $(SOURCES) | show-summary show-header
 	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
 ##TARGET clean: clean build
-clean:
+clean: clean-generated-files
 	@cargo clean
-	@rm -f $(GENERATED_FILES)
 	@rm -f tarpaulin-report.html
 	@rm -f $(CONFIGS)
 
+##TARGET clean-generated-files: clean generated files
+clean-generated-files:
+	@rm -f $(GENERATED_FILES)
+
 vendor:
 	@cargo vendor
 

src/runtime/Makefile

@@ -879,15 +879,16 @@ vendor: handle_vendor
 
 static-checks-build: $(GENERATED_FILES)
 
-clean:
+clean: clean-generated-files
 	$(QUIET_CLEAN)rm -f \
-		$(CONFIGS) \
-		$(GENERATED_FILES) \
 		$(MONITOR) \
 		$(SHIMV2) \
 		$(TARGET) \
 		.git-commit .git-commit.tmp
 
+clean-generated-files:
+	$(QUIET_CLEAN)rm -f $(GENERATED_FILES)
+
 show-usage: show-header
 	@printf "• Overview:\n"
 	@printf "\n"
@@ -904,6 +905,7 @@ show-usage: show-header
 	@printf "\tfast-test                  : run tests with failfast option.\n"
 	@printf "\tcheck                      : run code checks.\n"
 	@printf "\tclean                      : remove built files.\n"
+	@printf "\tclean-generated-files      : remove generated files.\n"
 	@printf "\tcontainerd-shim-v2         : only build containerd shim v2.\n"
 	@printf "\tcoverage                   : run coverage tests.\n"
 	@printf "\tdefault                    : same as 'make build' (or just 'make').\n"

tests/integration/kubernetes/k8s-measured-rootfs.bats

@@ -0,0 +1,52 @@
+#!/usr/bin/env bats
+#
+# Copyright (c) 2023 Red Hat
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+load "${BATS_TEST_DIRNAME}/lib.sh"
+load "${BATS_TEST_DIRNAME}/tests_common.sh"
+
+check_and_skip() {
+	# Currently the only kernel built with measured rootfs support is
+	# the kernel-tdx-experimental.
+	[ "${KATA_HYPERVISOR}" = "qemu-tdx" ] || \
+		skip "measured rootfs tests not implemented for hypervisor: $KATA_HYPERVISOR"
+}
+
+setup() {
+	check_and_skip
+	setup_common
+}
+
+teardown() {
+	check_and_skip
+
+	kubectl describe -f "${pod_config}" || true
+	kubectl delete -f "${pod_config}" || true
+}
+
+@test "Test cannnot launch pod with measured boot enabled and incorrect hash" {
+	pod_config="$(new_pod_config nginx "kata-${KATA_HYPERVISOR}")"
+
+	incorrect_hash="5180b1568c2ba972e4e06ee0a55976acae8329f2a5d1d2004395635e1ec4a76e"
+
+	# Despite the kernel being built with support, it is not currently enabled
+	# on configuration.toml. To avoid editing that file on the worker node,
+	# here it will be enabled via pod annotations.
+	set_metadata_annotation "$pod_config" \
+		"io.katacontainers.config.hypervisor.kernel_params" \
+		"rootfs_verity.scheme=dm-verity rootfs_verity.hash=$incorrect_hash"
+	# Run on a specific node so we know from where to inspect the logs
+	set_node "$pod_config" "$node"
+
+	# For debug sake
+	echo "Pod $pod_config file:"
+	cat $pod_config
+
+	assert_pod_fail "$pod_config"
+
+	assert_logs_contain "$node" kata "$node_start_time" \
+		'verity: .* metadata block .* is corrupted'
+}

tests/integration/kubernetes/lib.sh

@@ -0,0 +1,185 @@
+#!/bin/bash
+# Copyright (c) 2021, 2022 IBM Corporation
+# Copyright (c) 2022, 2023 Red Hat
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# This provides generic functions to use in the tests.
+#
+set -e
+
+# Delete all pods if any exist, otherwise just return
+#
+k8s_delete_all_pods_if_any_exists() {
+	[ -z "$(kubectl get --no-headers pods)" ] || \
+		kubectl delete --all pods
+}
+
+FIXTURES_DIR="${BATS_TEST_DIRNAME}/runtimeclass_workloads"
+
+# Wait until the pod is not 'Ready'. Fail if it hits the timeout.
+#
+# Parameters:
+#	$1 - the sandbox ID
+#	$2 - wait time in seconds. Defaults to 120. (optional)
+#
+k8s_wait_pod_be_ready() {
+	local pod_name="$1"
+	local wait_time="${2:-120}"
+
+	kubectl wait --timeout="${wait_time}s" --for=condition=ready "pods/$pod_name"
+}
+
+# Create a pod and wait it be ready, otherwise fail.
+#
+# Parameters:
+#	$1 - the pod configuration file.
+#
+k8s_create_pod() {
+	local config_file="$1"
+	local pod_name=""
+
+	if [ ! -f "${config_file}" ]; then
+		echo "Pod config file '${config_file}' does not exist"
+		return 1
+	fi
+
+	kubectl apply -f "${config_file}"
+	if ! pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}'); then
+		echo "Failed to create the pod"
+		return 1
+	fi
+
+	if ! k8s_wait_pod_be_ready "$pod_name"; then
+		# TODO: run this command for debugging. Maybe it should be
+		#       guarded by DEBUG=true?
+		kubectl get pods "$pod_name"
+		return 1
+	fi
+}
+
+# Check the logged messages on host have a given message.
+#
+# Parameters:
+#	$1 - the k8s worker node name
+#	$2 - the syslog identifier as in journalctl's -t option
+#	$3 - only logs since date/time (%Y-%m-%d %H:%M:%S)
+#	$4 - the message
+#
+assert_logs_contain() {
+	local node="$1"
+	local log_id="$2"
+	local datetime="$3"
+	local message="$4"
+
+	# Note: with image-rs we get more than the default 1000 lines of logs
+	print_node_journal "$node" "$log_id" --since "$datetime" -n 100000 \
+		grep "$message"
+}
+
+# Create a pod then assert it fails to run. Use in tests that you expect the
+# pod creation to fail.
+#
+# Note: a good testing practice is to afterwards check that the pod creation
+# failed because of the expected reason.
+#
+# Parameters:
+#	$1 - the pod configuration file.
+#
+assert_pod_fail() {
+	local container_config="$1"
+	echo "In assert_pod_fail: $container_config"
+
+	echo "Attempt to create the container but it should fail"
+	! k8s_create_pod "$container_config" || /bin/false
+}
+
+# Create a pod configuration out of a template file.
+#
+# Parameters:
+#	$1 - the container image.
+#	$2 - the runtimeclass
+#
+# Return:
+# 	the path to the configuration file. The caller should not care about
+# 	its removal afterwards as it is created under the bats temporary
+# 	directory.
+#
+new_pod_config() {
+	local base_config="${FIXTURES_DIR}/pod-config.yaml.in"
+	local image="$1"
+	local runtimeclass="$2"
+	local new_config
+
+	# The runtimeclass is not optional.
+	[ -n "$runtimeclass" ] || return 1
+
+	new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${base_config}").XXX")
+	IMAGE="$image" RUNTIMECLASS="$runtimeclass" envsubst < "$base_config" > "$new_config"
+	echo "$new_config"
+}
+
+# Set an annotation on configuration metadata.
+#
+# Usually you will pass a pod configuration file where the 'metadata'
+# is relative to the 'root' path. Other configuration files like deployments,
+# the annotation should be set on 'spec.template.metadata', so use the 4th
+# parameter of this function to pass the base metadata path (for deployments
+# cases, it will be 'spec.template' for example).
+#
+# Parameters:
+#	$1 - the yaml file
+#	$2 - the annotation key
+#	$3 - the annotation value
+#	$4 - (optional) base metadata path
+set_metadata_annotation() {
+	local yaml="${1}"
+	local key="${2}"
+	local value="${3}"
+	local metadata_path="${4:-}"
+	local annotation_key=""
+
+	[ -n "$metadata_path" ] && annotation_key+="${metadata_path}."
+
+	# yaml annotation key name.
+	annotation_key+="metadata.annotations.\"${key}\""
+
+	echo "$annotation_key"
+	# yq set annotations in yaml. Quoting the key because it can have
+	# dots.
+	yq w -i --style=double "${yaml}" "${annotation_key}" "${value}"
+}
+
+# Set the node name on configuration spec.
+#
+# Parameters:
+#	$1 - the yaml file
+#	$2 - the node name
+#
+set_node() {
+	local yaml="$1"
+	local node="$2"
+	[ -n "$node" ] || return 1
+
+	yq w -i "${yaml}" "spec.nodeName" "$node"
+}
+
+# Get the systemd's journal from a worker node
+#
+# Parameters:
+#	$1 - the k8s worker node name
+#	$2 - the syslog identifier as in journalctl's -t option
+#	$N - (optional) any extra parameters to journalctl
+#
+print_node_journal() {
+	local node="$1"
+	local id="$2"
+	shift 2
+	local img="quay.io/prometheus/busybox"
+
+	kubectl debug --image "$img" -q -it "node/${node}" \
+		-- chroot /host journalctl -x -t "$id" --no-pager "$@"
+	# Delete the debugger pod
+	kubectl get pods -o name | grep "node-debugger-${node}" | \
+		xargs kubectl delete > /dev/null
+}

tests/integration/kubernetes/run_kubernetes_tests.sh

@@ -38,6 +38,7 @@ else
 		"k8s-kill-all-process-in-container.bats" \
 		"k8s-limit-range.bats" \
 		"k8s-liveness-probes.bats" \
+		"k8s-measured-rootfs.bats" \
 		"k8s-memory.bats" \
 		"k8s-nested-configmap-secret.bats" \
 		"k8s-oom.bats" \

tests/integration/kubernetes/runtimeclass_workloads/pod-config.yaml.in

@@ -0,0 +1,14 @@
+# Copyright (c) 2021, 2022 IBM Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-e2e
+spec:
+  runtimeClassName: $RUNTIMECLASS
+  containers:
+  - name: test-container
+    image: $IMAGE
+    imagePullPolicy: Always

tests/integration/kubernetes/tests_common.sh

@@ -33,6 +33,23 @@ dragonball_limitations="https://github.com/kata-containers/kata-containers/issue
 # overwrite it.
 export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}"
 
+# Common setup for tests.
+#
+# Global variables exported:
+#	$node	             - random picked node that has kata installed
+#	$node_start_date     - start date/time at the $node for the sake of
+#                          fetching logs
+#
+setup_common() {
+	node=$(get_one_kata_node)
+	[ -n "$node" ]
+	node_start_time=$(exec_host "$node" date +\"%Y-%m-%d %H:%M:%S\")
+	[ -n "$node_start_time" ]
+	export node node_start_time
+
+	k8s_delete_all_pods_if_any_exists || true
+}
+
 get_pod_config_dir() {
 	pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads_work"
 	info "k8s configured to use runtimeclass"

tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh

@@ -25,7 +25,6 @@ readonly versions_yaml="${repo_root_dir}/versions.yaml"
 readonly agent_builder="${static_build_dir}/agent/build.sh"
 readonly clh_builder="${static_build_dir}/cloud-hypervisor/build-static-clh.sh"
 readonly firecracker_builder="${static_build_dir}/firecracker/build-static-firecracker.sh"
-readonly initramfs_builder="${static_build_dir}/initramfs/build.sh"
 readonly kernel_builder="${static_build_dir}/kernel/build.sh"
 readonly ovmf_builder="${static_build_dir}/ovmf/build.sh"
 readonly qemu_builder="${static_build_dir}/qemu/build-static-qemu.sh"
@@ -300,7 +299,7 @@ install_cached_kernel_tarball_component() {
 install_kernel_helper() {
 	local kernel_version_yaml_path="${1}"
 	local kernel_name="${2}"
-	local extra_cmd=${3}
+	local extra_cmd="${3:-}"
 
 	export kernel_version="$(get_from_kata_deps ${kernel_version_yaml_path})"
 	export kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)"
@@ -314,11 +313,6 @@ install_kernel_helper() {
 
 	install_cached_kernel_tarball_component ${kernel_name} ${module_dir} && return 0
 
-	if [ "${MEASURED_ROOTFS}" == "yes" ]; then
-		info "build initramfs for cc kernel"
-		"${initramfs_builder}"
-	fi
-
 	info "build ${kernel_name}"
 	info "Kernel version ${kernel_version}"
 	DESTDIR="${destdir}" PREFIX="${prefix}" "${kernel_builder}" -v "${kernel_version}" ${extra_cmd}
@@ -605,18 +599,7 @@ install_shimv2() {
 	export GO_VERSION
 	export RUST_VERSION
 
-	if [ "${MEASURED_ROOTFS}" == "yes" ]; then
-	        extra_opts="DEFSERVICEOFFLOAD=true"
-		if [ -f "${repo_root_dir}/tools/osbuilder/root_hash.txt" ]; then
-			root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder//root_hash.txt")
-			root_measure_config="rootfs_verity.scheme=dm-verity rootfs_verity.hash=${root_hash}"
-			extra_opts+=" ROOTMEASURECONFIG=\"${root_measure_config}\""
-		fi
-
-		DESTDIR="${destdir}" PREFIX="${prefix}" EXTRA_OPTS="${extra_opts}" "${shimv2_builder}"
-	else
 	DESTDIR="${destdir}" PREFIX="${prefix}" "${shimv2_builder}"
-	fi
 }
 
 install_ovmf() {

tools/packaging/kernel/build-kernel.sh

@@ -64,11 +64,11 @@ PREFIX="${PREFIX:-/usr}"
 kernel_url=""
 #Linux headers for GPU guest fs module building
 linux_headers=""
+# Enable measurement of the guest rootfs at boot.
+measured_rootfs="false"
 
 CROSS_BUILD_ARG=""
 
-MEASURED_ROOTFS=${MEASURED_ROOTFS:-no}
-
 packaging_scripts_dir="${script_dir}/../scripts"
 source "${packaging_scripts_dir}/lib.sh"
 
@@ -103,6 +103,7 @@ Options:
 	-g <vendor> 	: GPU vendor, intel or nvidia.
 	-h          	: Display this help.
 	-H <deb|rpm>	: Linux headers for guest fs module building.
+	-m              : Enable measured rootfs.
 	-k <path>   	: Path to kernel to build.
 	-p <path>   	: Path to a directory with patches to apply to kernel.
 	-s          	: Skip .config checks
@@ -127,6 +128,12 @@ arch_to_kernel() {
 	esac
 }
 
+# When building for measured rootfs the initramfs image should be previously built.
+check_initramfs_or_die() {
+	[ -f "${default_initramfs}" ] || \
+		die "Initramfs for measured rootfs not found at ${default_initramfs}"
+}
+
 get_tee_kernel() {
 	local version="${1}"
 	local kernel_path="${2}"
@@ -270,17 +277,16 @@ get_kernel_frag_path() {
 		all_configs="${all_configs} ${gpu_configs}"
 	fi
 
-	if [ "${MEASURED_ROOTFS}" == "yes" ]; then
+	if [ "${measured_rootfs}" == "true" ]; then
 		info "Enabling config for confidential guest trust storage protection"
 		local cryptsetup_configs="$(ls ${common_path}/confidential_containers/cryptsetup.conf)"
 		all_configs="${all_configs} ${cryptsetup_configs}"
 
-		if [ -f "${default_initramfs}" ]; then
+		check_initramfs_or_die
 		info "Enabling config for confidential guest measured boot"
 		local initramfs_configs="$(ls ${common_path}/confidential_containers/initramfs.conf)"
 		all_configs="${all_configs} ${initramfs_configs}"
 	fi
-	fi
 
 	if [[ "${conf_guest}" != "" ]];then
 		info "Enabling config for '${conf_guest}' confidential guest protection"
@@ -431,7 +437,8 @@ setup_kernel() {
 	[ -n "${hypervisor_target}" ] || hypervisor_target="kvm"
 	[ -n "${kernel_config_path}" ] || kernel_config_path=$(get_default_kernel_config "${kernel_version}" "${hypervisor_target}" "${arch_target}" "${kernel_path}")
 
-	if [ "${MEASURED_ROOTFS}" == "yes" ] && [ -f "${default_initramfs}" ]; then
+	if [ "${measured_rootfs}" == "true" ]; then
+		check_initramfs_or_die
 		info "Copying initramfs from: ${default_initramfs}"
 		cp "${default_initramfs}" ./
 	fi
@@ -538,7 +545,7 @@ install_kata() {
 }
 
 main() {
-	while getopts "a:b:c:deEfg:hH:k:p:t:u:v:x:" opt; do
+	while getopts "a:b:c:deEfg:hH:k:mp:t:u:v:x:" opt; do
 		case "$opt" in
 			a)
 				arch_target="${OPTARG}"
@@ -572,6 +579,9 @@ main() {
 			H)
 				linux_headers="${OPTARG}"
 				;;
+			m)
+				measured_rootfs="true"
+				;;
 			k)
 				kernel_path="$(realpath ${OPTARG})"
 				;;

tools/packaging/static-build/initramfs/init.sh

@@ -30,8 +30,24 @@ rootfs_hash=$(get_option rootfs_verity.hash)
 root_device=$(get_option root)
 hash_device=${root_device%?}2
 
-if [ -e ${root_device} ] && [ -e ${hash_device} ] && [ "${rootfs_verifier}" = "dm-verity" ]
+# The root device should exist to be either verified then mounted or
+# just mounted when verification is disabled.
+if [ ! -e "${root_device}" ]
 then
+	echo "No root device ${root_device} found"
+	exit 1
+fi
+
+if [ "${rootfs_verifier}" = "dm-verity" ]
+then
+	echo "Verify the root device with ${rootfs_verifier}"
+
+	if [ ! -e "${hash_device}" ]
+	then
+		echo "No hash device ${hash_device} found. Cannot verify the root device"
+		exit 1
+	fi
+
 	veritysetup open "${root_device}" root "${hash_device}" "${rootfs_hash}"
 	mount /dev/mapper/root /mnt
 else

tools/packaging/static-build/kernel/build.sh

@@ -13,6 +13,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${script_dir}/../../scripts/lib.sh"
 
 readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
+readonly initramfs_builder="${repo_root_dir}/tools/packaging/static-build/initramfs/build.sh"
 
 BUILDX=
 PLATFORM=
@@ -20,6 +21,16 @@ PLATFORM=
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
 container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}"
+MEASURED_ROOTFS=${MEASURED_ROOTFS:-no}
+kernel_builder_args="-a ${ARCH} $*"
+
+if [ "${MEASURED_ROOTFS}" == "yes" ]; then
+	info "build initramfs for cc kernel"
+	"${initramfs_builder}"
+	# Turn on the flag to build the kernel with support to
+	# measured rootfs.
+	kernel_builder_args+=" -m"
+fi
 
 if [ "${CROSS_BUILD}" == "true" ]; then
        container_image="${container_image}-${ARCH}-cross-build"
@@ -39,23 +50,22 @@ sudo docker pull ${container_image} || \
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \
-	--env MEASURED_ROOTFS="${MEASURED_ROOTFS:-}" \
 	"${container_image}" \
-	bash -c "${kernel_builder} -a ${ARCH} $* setup"
+	bash -c "${kernel_builder} ${kernel_builder_args} setup"
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \
 	"${container_image}" \
-	bash -c "${kernel_builder} -a ${ARCH} $* build"
+	bash -c "${kernel_builder} ${kernel_builder_args} build"
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \
 	--env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \
 	"${container_image}" \
-	bash -c "${kernel_builder} -a ${ARCH} $* install"
+	bash -c "${kernel_builder} ${kernel_builder_args} install"
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \
 	--env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \
 	"${container_image}" \
-	bash -c "${kernel_builder} -a ${ARCH} $* build-headers"
+	bash -c "${kernel_builder} ${kernel_builder_args} build-headers"

tools/packaging/static-build/shim-v2/build.sh

@@ -25,6 +25,19 @@ container_image="${SHIM_V2_CONTAINER_BUILDER:-$(get_shim_v2_image_name)}"
 EXTRA_OPTS="${EXTRA_OPTS:-""}"
 
 [ "${CROSS_BUILD}" == "true" ] && container_image_bk="${container_image}" && container_image="${container_image}-cross-build"
+if [ "${MEASURED_ROOTFS}" == "yes" ]; then
+	EXTRA_OPTS+=" DEFSERVICEOFFLOAD=true"
+	info "Enable rootfs measurement config"
+
+	root_hash_file="${repo_root_dir}/tools/osbuilder/root_hash.txt"
+	[ -f "$root_hash_file" ] || \
+		die "Root hash file for measured rootfs not found at ${root_hash_file}"
+
+	root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${root_hash_file}")
+	root_measure_config="rootfs_verity.scheme=dm-verity rootfs_verity.hash=${root_hash}"
+	EXTRA_OPTS+=" ROOTMEASURECONFIG=\"${root_measure_config}\""
+fi
+
 sudo docker pull ${container_image} || \
 	(sudo docker ${BUILDX} build ${PLATFORM}  \
 		--build-arg GO_VERSION="${GO_VERSION}" \
@@ -49,7 +62,8 @@ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	--env CC="${CC}" \
 	-w "${repo_root_dir}/src/runtime-rs" \
 	"${container_image}" \
-	bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX=${PREFIX} QEMUCMD=qemu-system-${arch}"
+	bash -c "git config --global --add safe.directory ${repo_root_dir} && \
+		make clean-generated-files && make PREFIX=${PREFIX} QEMUCMD=qemu-system-${arch}"
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	--env CROSS_BUILD=${CROSS_BUILD} \
@@ -64,7 +78,8 @@ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${repo_root_dir}/src/runtime" \
 	"${container_image}" \
-	bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX=${PREFIX} QEMUCMD=qemu-system-${arch} ${EXTRA_OPTS}"
+	bash -c "git config --global --add safe.directory ${repo_root_dir} && \
+		make clean-generated-files && make PREFIX=${PREFIX} QEMUCMD=qemu-system-${arch} ${EXTRA_OPTS}"
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${repo_root_dir}/src/runtime" \