diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index ddf857a54b..fef3ee3728 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -452,12 +452,15 @@ optimize: $(SOURCES) | show-summary show-header @RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES) ##TARGET clean: clean build -clean: +clean: clean-generated-files @cargo clean - @rm -f $(GENERATED_FILES) @rm -f tarpaulin-report.html @rm -f $(CONFIGS) +##TARGET clean-generated-files: clean generated files +clean-generated-files: + @rm -f $(GENERATED_FILES) + vendor: @cargo vendor diff --git a/src/runtime/Makefile b/src/runtime/Makefile index c9cf7ac584..2d96cd3baa 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -879,15 +879,16 @@ vendor: handle_vendor static-checks-build: $(GENERATED_FILES) -clean: +clean: clean-generated-files $(QUIET_CLEAN)rm -f \ - $(CONFIGS) \ - $(GENERATED_FILES) \ $(MONITOR) \ $(SHIMV2) \ $(TARGET) \ .git-commit .git-commit.tmp +clean-generated-files: + $(QUIET_CLEAN)rm -f $(GENERATED_FILES) + show-usage: show-header @printf "• Overview:\n" @printf "\n" @@ -904,6 +905,7 @@ show-usage: show-header @printf "\tfast-test : run tests with failfast option.\n" @printf "\tcheck : run code checks.\n" @printf "\tclean : remove built files.\n" + @printf "\tclean-generated-files : remove generated files.\n" @printf "\tcontainerd-shim-v2 : only build containerd shim v2.\n" @printf "\tcoverage : run coverage tests.\n" @printf "\tdefault : same as 'make build' (or just 'make').\n" diff --git a/tests/integration/kubernetes/k8s-measured-rootfs.bats b/tests/integration/kubernetes/k8s-measured-rootfs.bats new file mode 100644 index 0000000000..a98f96c6a8 --- /dev/null +++ b/tests/integration/kubernetes/k8s-measured-rootfs.bats @@ -0,0 +1,52 @@ +#!/usr/bin/env bats +# +# Copyright (c) 2023 Red Hat +# +# SPDX-License-Identifier: Apache-2.0 +# + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/tests_common.sh" + +check_and_skip() { + # Currently the only kernel built with measured rootfs support is + # the kernel-tdx-experimental. + [ "${KATA_HYPERVISOR}" = "qemu-tdx" ] || \ + skip "measured rootfs tests not implemented for hypervisor: $KATA_HYPERVISOR" +} + +setup() { + check_and_skip + setup_common +} + +teardown() { + check_and_skip + + kubectl describe -f "${pod_config}" || true + kubectl delete -f "${pod_config}" || true +} + +@test "Test cannnot launch pod with measured boot enabled and incorrect hash" { + pod_config="$(new_pod_config nginx "kata-${KATA_HYPERVISOR}")" + + incorrect_hash="5180b1568c2ba972e4e06ee0a55976acae8329f2a5d1d2004395635e1ec4a76e" + + # Despite the kernel being built with support, it is not currently enabled + # on configuration.toml. To avoid editing that file on the worker node, + # here it will be enabled via pod annotations. + set_metadata_annotation "$pod_config" \ + "io.katacontainers.config.hypervisor.kernel_params" \ + "rootfs_verity.scheme=dm-verity rootfs_verity.hash=$incorrect_hash" + # Run on a specific node so we know from where to inspect the logs + set_node "$pod_config" "$node" + + # For debug sake + echo "Pod $pod_config file:" + cat $pod_config + + assert_pod_fail "$pod_config" + + assert_logs_contain "$node" kata "$node_start_time" \ + 'verity: .* metadata block .* is corrupted' +} \ No newline at end of file diff --git a/tests/integration/kubernetes/lib.sh b/tests/integration/kubernetes/lib.sh new file mode 100644 index 0000000000..9b101a904a --- /dev/null +++ b/tests/integration/kubernetes/lib.sh @@ -0,0 +1,185 @@ +#!/bin/bash +# Copyright (c) 2021, 2022 IBM Corporation +# Copyright (c) 2022, 2023 Red Hat +# +# SPDX-License-Identifier: Apache-2.0 +# +# This provides generic functions to use in the tests. +# +set -e + +# Delete all pods if any exist, otherwise just return +# +k8s_delete_all_pods_if_any_exists() { + [ -z "$(kubectl get --no-headers pods)" ] || \ + kubectl delete --all pods +} + +FIXTURES_DIR="${BATS_TEST_DIRNAME}/runtimeclass_workloads" + +# Wait until the pod is not 'Ready'. Fail if it hits the timeout. +# +# Parameters: +# $1 - the sandbox ID +# $2 - wait time in seconds. Defaults to 120. (optional) +# +k8s_wait_pod_be_ready() { + local pod_name="$1" + local wait_time="${2:-120}" + + kubectl wait --timeout="${wait_time}s" --for=condition=ready "pods/$pod_name" +} + +# Create a pod and wait it be ready, otherwise fail. +# +# Parameters: +# $1 - the pod configuration file. +# +k8s_create_pod() { + local config_file="$1" + local pod_name="" + + if [ ! -f "${config_file}" ]; then + echo "Pod config file '${config_file}' does not exist" + return 1 + fi + + kubectl apply -f "${config_file}" + if ! pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}'); then + echo "Failed to create the pod" + return 1 + fi + + if ! k8s_wait_pod_be_ready "$pod_name"; then + # TODO: run this command for debugging. Maybe it should be + # guarded by DEBUG=true? + kubectl get pods "$pod_name" + return 1 + fi +} + +# Check the logged messages on host have a given message. +# +# Parameters: +# $1 - the k8s worker node name +# $2 - the syslog identifier as in journalctl's -t option +# $3 - only logs since date/time (%Y-%m-%d %H:%M:%S) +# $4 - the message +# +assert_logs_contain() { + local node="$1" + local log_id="$2" + local datetime="$3" + local message="$4" + + # Note: with image-rs we get more than the default 1000 lines of logs + print_node_journal "$node" "$log_id" --since "$datetime" -n 100000 \ + grep "$message" +} + +# Create a pod then assert it fails to run. Use in tests that you expect the +# pod creation to fail. +# +# Note: a good testing practice is to afterwards check that the pod creation +# failed because of the expected reason. +# +# Parameters: +# $1 - the pod configuration file. +# +assert_pod_fail() { + local container_config="$1" + echo "In assert_pod_fail: $container_config" + + echo "Attempt to create the container but it should fail" + ! k8s_create_pod "$container_config" || /bin/false +} + +# Create a pod configuration out of a template file. +# +# Parameters: +# $1 - the container image. +# $2 - the runtimeclass +# +# Return: +# the path to the configuration file. The caller should not care about +# its removal afterwards as it is created under the bats temporary +# directory. +# +new_pod_config() { + local base_config="${FIXTURES_DIR}/pod-config.yaml.in" + local image="$1" + local runtimeclass="$2" + local new_config + + # The runtimeclass is not optional. + [ -n "$runtimeclass" ] || return 1 + + new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename "${base_config}").XXX") + IMAGE="$image" RUNTIMECLASS="$runtimeclass" envsubst < "$base_config" > "$new_config" + echo "$new_config" +} + +# Set an annotation on configuration metadata. +# +# Usually you will pass a pod configuration file where the 'metadata' +# is relative to the 'root' path. Other configuration files like deployments, +# the annotation should be set on 'spec.template.metadata', so use the 4th +# parameter of this function to pass the base metadata path (for deployments +# cases, it will be 'spec.template' for example). +# +# Parameters: +# $1 - the yaml file +# $2 - the annotation key +# $3 - the annotation value +# $4 - (optional) base metadata path +set_metadata_annotation() { + local yaml="${1}" + local key="${2}" + local value="${3}" + local metadata_path="${4:-}" + local annotation_key="" + + [ -n "$metadata_path" ] && annotation_key+="${metadata_path}." + + # yaml annotation key name. + annotation_key+="metadata.annotations.\"${key}\"" + + echo "$annotation_key" + # yq set annotations in yaml. Quoting the key because it can have + # dots. + yq w -i --style=double "${yaml}" "${annotation_key}" "${value}" +} + +# Set the node name on configuration spec. +# +# Parameters: +# $1 - the yaml file +# $2 - the node name +# +set_node() { + local yaml="$1" + local node="$2" + [ -n "$node" ] || return 1 + + yq w -i "${yaml}" "spec.nodeName" "$node" +} + +# Get the systemd's journal from a worker node +# +# Parameters: +# $1 - the k8s worker node name +# $2 - the syslog identifier as in journalctl's -t option +# $N - (optional) any extra parameters to journalctl +# +print_node_journal() { + local node="$1" + local id="$2" + shift 2 + local img="quay.io/prometheus/busybox" + + kubectl debug --image "$img" -q -it "node/${node}" \ + -- chroot /host journalctl -x -t "$id" --no-pager "$@" + # Delete the debugger pod + kubectl get pods -o name | grep "node-debugger-${node}" | \ + xargs kubectl delete > /dev/null +} diff --git a/tests/integration/kubernetes/run_kubernetes_tests.sh b/tests/integration/kubernetes/run_kubernetes_tests.sh index b16c22ae64..40500c237d 100644 --- a/tests/integration/kubernetes/run_kubernetes_tests.sh +++ b/tests/integration/kubernetes/run_kubernetes_tests.sh @@ -38,6 +38,7 @@ else "k8s-kill-all-process-in-container.bats" \ "k8s-limit-range.bats" \ "k8s-liveness-probes.bats" \ + "k8s-measured-rootfs.bats" \ "k8s-memory.bats" \ "k8s-nested-configmap-secret.bats" \ "k8s-oom.bats" \ diff --git a/tests/integration/kubernetes/runtimeclass_workloads/pod-config.yaml.in b/tests/integration/kubernetes/runtimeclass_workloads/pod-config.yaml.in new file mode 100644 index 0000000000..b79032d445 --- /dev/null +++ b/tests/integration/kubernetes/runtimeclass_workloads/pod-config.yaml.in @@ -0,0 +1,14 @@ +# Copyright (c) 2021, 2022 IBM Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v1 +kind: Pod +metadata: + name: test-e2e +spec: + runtimeClassName: $RUNTIMECLASS + containers: + - name: test-container + image: $IMAGE + imagePullPolicy: Always diff --git a/tests/integration/kubernetes/tests_common.sh b/tests/integration/kubernetes/tests_common.sh index 922286ada5..b5fef531d9 100644 --- a/tests/integration/kubernetes/tests_common.sh +++ b/tests/integration/kubernetes/tests_common.sh @@ -33,6 +33,23 @@ dragonball_limitations="https://github.com/kata-containers/kata-containers/issue # overwrite it. export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}" +# Common setup for tests. +# +# Global variables exported: +# $node - random picked node that has kata installed +# $node_start_date - start date/time at the $node for the sake of +# fetching logs +# +setup_common() { + node=$(get_one_kata_node) + [ -n "$node" ] + node_start_time=$(exec_host "$node" date +\"%Y-%m-%d %H:%M:%S\") + [ -n "$node_start_time" ] + export node node_start_time + + k8s_delete_all_pods_if_any_exists || true +} + get_pod_config_dir() { pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads_work" info "k8s configured to use runtimeclass" diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index cb93fd1a15..67d2b22e65 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -25,7 +25,6 @@ readonly versions_yaml="${repo_root_dir}/versions.yaml" readonly agent_builder="${static_build_dir}/agent/build.sh" readonly clh_builder="${static_build_dir}/cloud-hypervisor/build-static-clh.sh" readonly firecracker_builder="${static_build_dir}/firecracker/build-static-firecracker.sh" -readonly initramfs_builder="${static_build_dir}/initramfs/build.sh" readonly kernel_builder="${static_build_dir}/kernel/build.sh" readonly ovmf_builder="${static_build_dir}/ovmf/build.sh" readonly qemu_builder="${static_build_dir}/qemu/build-static-qemu.sh" @@ -300,7 +299,7 @@ install_cached_kernel_tarball_component() { install_kernel_helper() { local kernel_version_yaml_path="${1}" local kernel_name="${2}" - local extra_cmd=${3} + local extra_cmd="${3:-}" export kernel_version="$(get_from_kata_deps ${kernel_version_yaml_path})" export kernel_kata_config_version="$(cat ${repo_root_dir}/tools/packaging/kernel/kata_config_version)" @@ -314,11 +313,6 @@ install_kernel_helper() { install_cached_kernel_tarball_component ${kernel_name} ${module_dir} && return 0 - if [ "${MEASURED_ROOTFS}" == "yes" ]; then - info "build initramfs for cc kernel" - "${initramfs_builder}" - fi - info "build ${kernel_name}" info "Kernel version ${kernel_version}" DESTDIR="${destdir}" PREFIX="${prefix}" "${kernel_builder}" -v "${kernel_version}" ${extra_cmd} @@ -605,18 +599,7 @@ install_shimv2() { export GO_VERSION export RUST_VERSION - if [ "${MEASURED_ROOTFS}" == "yes" ]; then - extra_opts="DEFSERVICEOFFLOAD=true" - if [ -f "${repo_root_dir}/tools/osbuilder/root_hash.txt" ]; then - root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${repo_root_dir}/tools/osbuilder//root_hash.txt") - root_measure_config="rootfs_verity.scheme=dm-verity rootfs_verity.hash=${root_hash}" - extra_opts+=" ROOTMEASURECONFIG=\"${root_measure_config}\"" - fi - - DESTDIR="${destdir}" PREFIX="${prefix}" EXTRA_OPTS="${extra_opts}" "${shimv2_builder}" - else - DESTDIR="${destdir}" PREFIX="${prefix}" "${shimv2_builder}" - fi + DESTDIR="${destdir}" PREFIX="${prefix}" "${shimv2_builder}" } install_ovmf() { diff --git a/tools/packaging/kernel/build-kernel.sh b/tools/packaging/kernel/build-kernel.sh index 8119be7047..f1d1bb62c2 100755 --- a/tools/packaging/kernel/build-kernel.sh +++ b/tools/packaging/kernel/build-kernel.sh @@ -64,11 +64,11 @@ PREFIX="${PREFIX:-/usr}" kernel_url="" #Linux headers for GPU guest fs module building linux_headers="" +# Enable measurement of the guest rootfs at boot. +measured_rootfs="false" CROSS_BUILD_ARG="" -MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} - packaging_scripts_dir="${script_dir}/../scripts" source "${packaging_scripts_dir}/lib.sh" @@ -103,6 +103,7 @@ Options: -g : GPU vendor, intel or nvidia. -h : Display this help. -H : Linux headers for guest fs module building. + -m : Enable measured rootfs. -k : Path to kernel to build. -p : Path to a directory with patches to apply to kernel. -s : Skip .config checks @@ -127,6 +128,12 @@ arch_to_kernel() { esac } +# When building for measured rootfs the initramfs image should be previously built. +check_initramfs_or_die() { + [ -f "${default_initramfs}" ] || \ + die "Initramfs for measured rootfs not found at ${default_initramfs}" +} + get_tee_kernel() { local version="${1}" local kernel_path="${2}" @@ -270,16 +277,15 @@ get_kernel_frag_path() { all_configs="${all_configs} ${gpu_configs}" fi - if [ "${MEASURED_ROOTFS}" == "yes" ]; then + if [ "${measured_rootfs}" == "true" ]; then info "Enabling config for confidential guest trust storage protection" local cryptsetup_configs="$(ls ${common_path}/confidential_containers/cryptsetup.conf)" all_configs="${all_configs} ${cryptsetup_configs}" - if [ -f "${default_initramfs}" ]; then - info "Enabling config for confidential guest measured boot" - local initramfs_configs="$(ls ${common_path}/confidential_containers/initramfs.conf)" - all_configs="${all_configs} ${initramfs_configs}" - fi + check_initramfs_or_die + info "Enabling config for confidential guest measured boot" + local initramfs_configs="$(ls ${common_path}/confidential_containers/initramfs.conf)" + all_configs="${all_configs} ${initramfs_configs}" fi if [[ "${conf_guest}" != "" ]];then @@ -431,7 +437,8 @@ setup_kernel() { [ -n "${hypervisor_target}" ] || hypervisor_target="kvm" [ -n "${kernel_config_path}" ] || kernel_config_path=$(get_default_kernel_config "${kernel_version}" "${hypervisor_target}" "${arch_target}" "${kernel_path}") - if [ "${MEASURED_ROOTFS}" == "yes" ] && [ -f "${default_initramfs}" ]; then + if [ "${measured_rootfs}" == "true" ]; then + check_initramfs_or_die info "Copying initramfs from: ${default_initramfs}" cp "${default_initramfs}" ./ fi @@ -538,7 +545,7 @@ install_kata() { } main() { - while getopts "a:b:c:deEfg:hH:k:p:t:u:v:x:" opt; do + while getopts "a:b:c:deEfg:hH:k:mp:t:u:v:x:" opt; do case "$opt" in a) arch_target="${OPTARG}" @@ -572,6 +579,9 @@ main() { H) linux_headers="${OPTARG}" ;; + m) + measured_rootfs="true" + ;; k) kernel_path="$(realpath ${OPTARG})" ;; diff --git a/tools/packaging/static-build/initramfs/init.sh b/tools/packaging/static-build/initramfs/init.sh index 4b224280b7..302ff475b2 100755 --- a/tools/packaging/static-build/initramfs/init.sh +++ b/tools/packaging/static-build/initramfs/init.sh @@ -30,8 +30,24 @@ rootfs_hash=$(get_option rootfs_verity.hash) root_device=$(get_option root) hash_device=${root_device%?}2 -if [ -e ${root_device} ] && [ -e ${hash_device} ] && [ "${rootfs_verifier}" = "dm-verity" ] +# The root device should exist to be either verified then mounted or +# just mounted when verification is disabled. +if [ ! -e "${root_device}" ] then + echo "No root device ${root_device} found" + exit 1 +fi + +if [ "${rootfs_verifier}" = "dm-verity" ] +then + echo "Verify the root device with ${rootfs_verifier}" + + if [ ! -e "${hash_device}" ] + then + echo "No hash device ${hash_device} found. Cannot verify the root device" + exit 1 + fi + veritysetup open "${root_device}" root "${hash_device}" "${rootfs_hash}" mount /dev/mapper/root /mnt else diff --git a/tools/packaging/static-build/kernel/build.sh b/tools/packaging/static-build/kernel/build.sh index b8deea4f09..a7bb4b4b0e 100755 --- a/tools/packaging/static-build/kernel/build.sh +++ b/tools/packaging/static-build/kernel/build.sh @@ -13,6 +13,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${script_dir}/../../scripts/lib.sh" readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh" +readonly initramfs_builder="${repo_root_dir}/tools/packaging/static-build/initramfs/build.sh" BUILDX= PLATFORM= @@ -20,6 +21,16 @@ PLATFORM= DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}" +MEASURED_ROOTFS=${MEASURED_ROOTFS:-no} +kernel_builder_args="-a ${ARCH} $*" + +if [ "${MEASURED_ROOTFS}" == "yes" ]; then + info "build initramfs for cc kernel" + "${initramfs_builder}" + # Turn on the flag to build the kernel with support to + # measured rootfs. + kernel_builder_args+=" -m" +fi if [ "${CROSS_BUILD}" == "true" ]; then container_image="${container_image}-${ARCH}-cross-build" @@ -39,23 +50,22 @@ sudo docker pull ${container_image} || \ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${PWD}" \ - --env MEASURED_ROOTFS="${MEASURED_ROOTFS:-}" \ "${container_image}" \ - bash -c "${kernel_builder} -a ${ARCH} $* setup" + bash -c "${kernel_builder} ${kernel_builder_args} setup" sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${PWD}" \ "${container_image}" \ - bash -c "${kernel_builder} -a ${ARCH} $* build" + bash -c "${kernel_builder} ${kernel_builder_args} build" sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${PWD}" \ --env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \ "${container_image}" \ - bash -c "${kernel_builder} -a ${ARCH} $* install" + bash -c "${kernel_builder} ${kernel_builder_args} install" sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${PWD}" \ --env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \ "${container_image}" \ - bash -c "${kernel_builder} -a ${ARCH} $* build-headers" + bash -c "${kernel_builder} ${kernel_builder_args} build-headers" diff --git a/tools/packaging/static-build/shim-v2/build.sh b/tools/packaging/static-build/shim-v2/build.sh index b702c2e157..f37cb91f40 100755 --- a/tools/packaging/static-build/shim-v2/build.sh +++ b/tools/packaging/static-build/shim-v2/build.sh @@ -25,6 +25,19 @@ container_image="${SHIM_V2_CONTAINER_BUILDER:-$(get_shim_v2_image_name)}" EXTRA_OPTS="${EXTRA_OPTS:-""}" [ "${CROSS_BUILD}" == "true" ] && container_image_bk="${container_image}" && container_image="${container_image}-cross-build" +if [ "${MEASURED_ROOTFS}" == "yes" ]; then + EXTRA_OPTS+=" DEFSERVICEOFFLOAD=true" + info "Enable rootfs measurement config" + + root_hash_file="${repo_root_dir}/tools/osbuilder/root_hash.txt" + [ -f "$root_hash_file" ] || \ + die "Root hash file for measured rootfs not found at ${root_hash_file}" + + root_hash=$(sudo sed -e 's/Root hash:\s*//g;t;d' "${root_hash_file}") + root_measure_config="rootfs_verity.scheme=dm-verity rootfs_verity.hash=${root_hash}" + EXTRA_OPTS+=" ROOTMEASURECONFIG=\"${root_measure_config}\"" +fi + sudo docker pull ${container_image} || \ (sudo docker ${BUILDX} build ${PLATFORM} \ --build-arg GO_VERSION="${GO_VERSION}" \ @@ -49,7 +62,8 @@ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ --env CC="${CC}" \ -w "${repo_root_dir}/src/runtime-rs" \ "${container_image}" \ - bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX=${PREFIX} QEMUCMD=qemu-system-${arch}" + bash -c "git config --global --add safe.directory ${repo_root_dir} && \ + make clean-generated-files && make PREFIX=${PREFIX} QEMUCMD=qemu-system-${arch}" sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ --env CROSS_BUILD=${CROSS_BUILD} \ @@ -64,7 +78,8 @@ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${repo_root_dir}/src/runtime" \ "${container_image}" \ - bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX=${PREFIX} QEMUCMD=qemu-system-${arch} ${EXTRA_OPTS}" + bash -c "git config --global --add safe.directory ${repo_root_dir} && \ + make clean-generated-files && make PREFIX=${PREFIX} QEMUCMD=qemu-system-${arch} ${EXTRA_OPTS}" sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ -w "${repo_root_dir}/src/runtime" \