From 023c4a17cff82b547426932f4bc53e89c7817d03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 8 Nov 2023 20:53:15 +0100 Subject: [PATCH 1/2] kata-deploy: Allow users to set hypervisor annotations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently the only way one can specify allowed hypervisor annotations is during build time, which is a big issue for users grabbing kata-deploy as we provide. Fixes: #8403 Signed-off-by: Fabiano Fidêncio --- .../kata-deploy/base/kata-deploy.yaml | 2 ++ .../kata-deploy/scripts/kata-deploy.sh | 33 +++++++++++++------ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml index 3b4e8888a3..8dde8962e7 100644 --- a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml @@ -39,6 +39,8 @@ spec: value: "false" - name: CREATE_DEFAULT_RUNTIMECLASS value: "false" + - name: ALLOWED_HYPERVISOR_ANNOTATIONS + value: "" securityContext: privileged: true volumeMounts: diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index 9d60a7ad04..da4c7adc89 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -15,9 +15,15 @@ containerd_conf_file="/etc/containerd/config.toml" containerd_conf_file_backup="${containerd_conf_file}.bak" IFS=' ' read -a shims <<< "$SHIMS" - default_shim="$DEFAULT_SHIM" +IFS=' ' read -a non_formatted_allowed_hypervisor_annotations <<< "$ALLOWED_HYPERVISOR_ANNOTATIONS" +allowed_hypervisor_annotations="" +for allowed_hypervisor_annotation in "${non_formatted_allowed_hypervisor_annotations[@]}"; do + allowed_hypervisor_annotations+="\"$allowed_hypervisor_annotation\", " +done +allowed_hypervisor_annotations=$(echo $allowed_hypervisor_annotations | sed 's/,$//') + # If we fail for any reason a message will be displayed die() { msg="$*" @@ -105,15 +111,20 @@ function install_artifacts() { [ -d /opt/kata/runtime-rs/bin ] && \ chmod +x /opt/kata/runtime-rs/bin/* - # Allow enabling debug for Kata Containers - if [[ "${DEBUG}" == "true" ]]; then - config_path="/opt/kata/share/defaults/kata-containers/" - for shim in "${shims[@]}"; do - sed -i -e 's/^#\(enable_debug\).*=.*$/\1 = true/g' "${config_path}/configuration-${shim}.toml" - sed -i -e 's/^#\(debug_console_enabled\).*=.*$/\1 = true/g' "${config_path}/configuration-${shim}.toml" - sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.log=debug initcall_debug"/g' "${config_path}/configuration-${shim}.toml" - done - fi + config_path="/opt/kata/share/defaults/kata-containers/" + for shim in "${shims[@]}"; do + local kata_config_file="${config_path}/configuration-${shim}.toml" + # Allow enabling debug for Kata Containers + if [[ "${DEBUG}" == "true" ]]; then + sed -i -e 's/^#\(enable_debug\).*=.*$/\1 = true/g' "${kata_config_file}" + sed -i -e 's/^#\(debug_console_enabled\).*=.*$/\1 = true/g' "${kata_config_file}" + sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.log=debug initcall_debug"/g' "${kata_config_file}" + fi + + if [ -n "${allowed_hypervisor_annotations}" ]; then + sed -i -e "s/^enable_annotations = \[\(.*\)\]/enable_annotations = [\1, $allowed_hypervisor_annotations]/" "${kata_config_file}" + fi + done # Allow Mariner to use custom configuration. if [ "${HOST_OS:-}" == "cbl-mariner" ]; then @@ -124,6 +135,7 @@ function install_artifacts() { sed -i -E "s|(path) = \".+/cloud-hypervisor\"|\1 = \"${clh_path}\"|" "${config_path}" fi + if [[ "${CREATE_RUNTIMECLASSES}" == "true" ]]; then create_runtimeclasses fi @@ -415,6 +427,7 @@ function main() { echo "* DEFAULT_SHIM: ${DEFAULT_SHIM}" echo "* CREATE_RUNTIMECLASSES: ${CREATE_RUNTIMECLASSES}" echo "* CREATE_DEFAULT_RUNTIMECLASS: ${CREATE_DEFAULT_RUNTIMECLASS}" + echo "* ALLOWED_HYPERVISOR_ANNOTATIONS: ${ALLOWED_HYPERVISOR_ANNOTATIONS}" # script requires that user is root euid=$(id -u) From 1a81989d20f23d38d55f5d0655fc5f3065408108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 8 Nov 2023 21:01:25 +0100 Subject: [PATCH 2/2] tests: k8s: Use the "ALLOWED_HYPERVISOR_ANNOTATIONS" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current kata-deploy code has been doing a `sed` to add allowed hypervisor annotations, so CBL mariner can be tested with their own kernel and initrd. Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/gha-run.sh | 1 + tools/packaging/kata-deploy/scripts/kata-deploy.sh | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 2c18c66e87..7dd65644a3 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -119,6 +119,7 @@ function deploy_kata() { yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[5].value' --tag '!!str' "true" if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then + yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[6].value' "initrd kernel" yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[+].name' "HOST_OS" yq write -i "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" 'spec.template.spec.containers[0].env[-1].value' "${KATA_HOST_OS}" fi diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index da4c7adc89..b0ce40e607 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -130,7 +130,6 @@ function install_artifacts() { if [ "${HOST_OS:-}" == "cbl-mariner" ]; then config_path="/opt/kata/share/defaults/kata-containers/configuration-clh.toml" clh_path="/opt/kata/bin/cloud-hypervisor-glibc" - sed -i -E 's|(enable_annotations) = .+|\1 = ["enable_iommu", "initrd", "kernel"]|' "${config_path}" sed -i -E "s|(valid_hypervisor_paths) = .+|\1 = [\"${clh_path}\"]|" "${config_path}" sed -i -E "s|(path) = \".+/cloud-hypervisor\"|\1 = \"${clh_path}\"|" "${config_path}" fi