From 6ce5e62c483e505c9670035bf7ab8980ae1de2cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 22 Aug 2024 12:09:57 +0200 Subject: [PATCH 1/3] kata-deploy: Add a $dest_dir var MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As we build our binaries with the `/opt/kata` prefix, that's the value of $dest_dir. Later in thise series it'll become handy, as we'll introduce a way to install the Kata Containers artefacts in a different location. Signed-off-by: Fabiano Fidêncio --- .../kata-deploy/scripts/kata-deploy.sh | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index 64b95493cb..5e5afcd3d2 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -36,6 +36,8 @@ AGENT_NO_PROXY="${AGENT_NO_PROXY:-}" PULL_TYPE_MAPPING="${PULL_TYPE_MAPPING:-}" IFS=',' read -a pull_types <<< "$PULL_TYPE_MAPPING" +dest_dir="/opt/kata" + # If we fail for any reason a message will be displayed die() { msg="$*" @@ -130,7 +132,7 @@ function get_kata_containers_config_path() { local shim="$1" # Directory holding pristine configuration files for the current default golang runtime. - local golang_config_path="/opt/kata/share/defaults/kata-containers/" + local golang_config_path="${dest_dir}/share/defaults/kata-containers/" # Directory holding pristine configuration files for the new rust runtime. # @@ -168,10 +170,10 @@ function get_kata_containers_runtime_path() { local runtime_path case "$shim" in cloud-hypervisor | dragonball | qemu-runtime-rs) - runtime_path="/opt/kata/runtime-rs/bin/containerd-shim-kata-v2" + runtime_path="${dest_dir}/runtime-rs/bin/containerd-shim-kata-v2" ;; *) - runtime_path="/opt/kata/bin/containerd-shim-kata-v2" + runtime_path="${dest_dir}/bin/containerd-shim-kata-v2" ;; esac @@ -237,10 +239,10 @@ function get_tdx_ovmf_path_from_distro() { function install_artifacts() { echo "copying kata artifacts onto host" - cp -au /opt/kata-artifacts/opt/kata/* /opt/kata/ - chmod +x /opt/kata/bin/* - [ -d /opt/kata/runtime-rs/bin ] && \ - chmod +x /opt/kata/runtime-rs/bin/* + cp -au /opt/kata-artifacts/opt/kata/* ${dest_dir}/ + chmod +x ${dest_dir}/bin/* + [ -d ${dest_dir}/runtime-rs/bin ] && \ + chmod +x ${dest_dir}/runtime-rs/bin/* local config_path @@ -302,8 +304,8 @@ function install_artifacts() { # Allow Mariner to use custom configuration. if [ "${HOST_OS:-}" == "cbl-mariner" ]; then - config_path="/opt/kata/share/defaults/kata-containers/configuration-clh.toml" - clh_path="/opt/kata/bin/cloud-hypervisor-glibc" + config_path="${dest_dir}/share/defaults/kata-containers/configuration-clh.toml" + clh_path="${dest_dir}/bin/cloud-hypervisor-glibc" sed -i -E "s|(valid_hypervisor_paths) = .+|\1 = [\"${clh_path}\"]|" "${config_path}" sed -i -E "s|(path) = \".+/cloud-hypervisor\"|\1 = \"${clh_path}\"|" "${config_path}" fi @@ -481,7 +483,7 @@ function configure_containerd() { function remove_artifacts() { echo "deleting kata artifacts" - rm -rf /opt/kata/* + rm -rf ${dest_dir}/* if [[ "${CREATE_RUNTIMECLASSES}" == "true" ]]; then delete_runtimeclasses From 7be77ebee54e0794ca25e10edb8008ededc83def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 22 Aug 2024 12:26:09 +0200 Subject: [PATCH 2/3] kata-deploy: helm: Stop mounting /opt/kata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's simply easier if we just use /host/opt/kata instead in our scripts, which will simplify a lot the logic of adding an INSTALLATION_PREFIX later on. Signed-off-by: Fabiano Fidêncio --- .../kata-deploy/templates/kata-deploy.yaml | 6 ------ .../kata-deploy/base/kata-deploy.yaml | 6 ------ .../kata-deploy/scripts/kata-deploy.sh | 18 +++++++++++------- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml index 22b5e13d79..cd64dfbd7e 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml @@ -63,8 +63,6 @@ spec: mountPath: /etc/crio/ - name: containerd-conf mountPath: /etc/containerd/ - - name: kata-artifacts - mountPath: /opt/kata/ - name: host mountPath: /host/ volumes: @@ -74,10 +72,6 @@ spec: - name: containerd-conf hostPath: path: '{{- template "containerdConfPath" .Values }}' - - name: kata-artifacts - hostPath: - path: /opt/kata/ - type: DirectoryOrCreate - name: host hostPath: path: / diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml index 9bf051fa79..19b0a381db 100644 --- a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml @@ -57,8 +57,6 @@ spec: mountPath: /etc/crio/ - name: containerd-conf mountPath: /etc/containerd/ - - name: kata-artifacts - mountPath: /opt/kata/ - name: host mountPath: /host/ volumes: @@ -68,10 +66,6 @@ spec: - name: containerd-conf hostPath: path: /etc/containerd/ - - name: kata-artifacts - hostPath: - path: /opt/kata/ - type: DirectoryOrCreate - name: host hostPath: path: / diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index 5e5afcd3d2..efce08c152 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -37,6 +37,7 @@ PULL_TYPE_MAPPING="${PULL_TYPE_MAPPING:-}" IFS=',' read -a pull_types <<< "$PULL_TYPE_MAPPING" dest_dir="/opt/kata" +host_install_dir="/host/${dest_dir}" # If we fail for any reason a message will be displayed die() { @@ -239,15 +240,17 @@ function get_tdx_ovmf_path_from_distro() { function install_artifacts() { echo "copying kata artifacts onto host" - cp -au /opt/kata-artifacts/opt/kata/* ${dest_dir}/ - chmod +x ${dest_dir}/bin/* - [ -d ${dest_dir}/runtime-rs/bin ] && \ - chmod +x ${dest_dir}/runtime-rs/bin/* + + mkdir -p ${host_install_dir} + cp -au /opt/kata-artifacts/opt/kata/* ${host_install_dir}/ + chmod +x ${host_install_dir}/bin/* + [ -d ${host_install_dir}/runtime-rs/bin ] && \ + chmod +x ${host_install_dir}/runtime-rs/bin/* local config_path for shim in "${shims[@]}"; do - config_path=$(get_kata_containers_config_path "${shim}") + config_path="/host/$(get_kata_containers_config_path "${shim}")" mkdir -p "$config_path" local kata_config_file="${config_path}/configuration-${shim}.toml" @@ -304,7 +307,7 @@ function install_artifacts() { # Allow Mariner to use custom configuration. if [ "${HOST_OS:-}" == "cbl-mariner" ]; then - config_path="${dest_dir}/share/defaults/kata-containers/configuration-clh.toml" + config_path="${host_install_dir}/share/defaults/kata-containers/configuration-clh.toml" clh_path="${dest_dir}/bin/cloud-hypervisor-glibc" sed -i -E "s|(valid_hypervisor_paths) = .+|\1 = [\"${clh_path}\"]|" "${config_path}" sed -i -E "s|(path) = \".+/cloud-hypervisor\"|\1 = \"${clh_path}\"|" "${config_path}" @@ -483,7 +486,8 @@ function configure_containerd() { function remove_artifacts() { echo "deleting kata artifacts" - rm -rf ${dest_dir}/* + + rm -rf ${host_install_dir} if [[ "${CREATE_RUNTIMECLASSES}" == "true" ]]; then delete_runtimeclasses From 0cb93ed1bb3df79343782d3aafc7dcb4e346e999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 22 Aug 2024 12:21:40 +0200 Subject: [PATCH 3/3] kata-deploy: helm: Add INSTALLATION_PREFIX option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will allow users to properly set the INSTALLATION_PREFIX when deploying Kata Containers. Signed-off-by: Fabiano Fidêncio --- .../kata-deploy/templates/kata-deploy.yaml | 2 + .../helm-chart/kata-deploy/values.yaml | 1 + .../kata-deploy/base/kata-deploy.yaml | 2 + .../kata-deploy/scripts/kata-deploy.sh | 61 ++++++++++++++++++- 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml index cd64dfbd7e..5d339e47b5 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/kata-deploy.yaml @@ -52,6 +52,8 @@ spec: value: {{ .Values.env.agentNoProxy | quote }} - name: PULL_TYPE_MAPPING value: {{ .Values.env.pullTypeMapping | quote }} + - name: INSTALLATION_PREFIX + value: {{ .Values.env.installationPrefix | quote }} {{- with .Values.env.hostOS }} - name: HOST_OS value: {{ . | quote }} diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml index b1f195d1f1..a59fb51068 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml @@ -16,4 +16,5 @@ env: agentHttpsProxy: "" agentNoProxy: "" pullTypeMapping: "" + installationPrefix: "" hostOS: "" diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml index 19b0a381db..5f5f9d93bb 100644 --- a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml @@ -50,6 +50,8 @@ spec: value: "" - name: PULL_TYPE_MAPPING value: "" + - name: INSTALLATION_PREFIX + value: "" securityContext: privileged: true volumeMounts: diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index efce08c152..03e602b1a0 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -36,8 +36,17 @@ AGENT_NO_PROXY="${AGENT_NO_PROXY:-}" PULL_TYPE_MAPPING="${PULL_TYPE_MAPPING:-}" IFS=',' read -a pull_types <<< "$PULL_TYPE_MAPPING" -dest_dir="/opt/kata" -host_install_dir="/host/${dest_dir}" +INSTALLATION_PREFIX="${INSTALLATION_PREFIX:-}" +default_dest_dir="/opt/kata" +dest_dir="${default_dest_dir}" +if [ -n "${INSTALLATION_PREFIX}" ]; then + # There's no `/` in between ${INSTALLATION_PREFIX} and ${default_dest_dir} + # as, otherwise, we'd have it doubled there, as: `/foo/bar//opt/kata` + dest_dir="${INSTALLATION_PREFIX}${default_dest_dir}" +fi +# Here, again, there's no `/` between /host and ${dest_dir}, otherwise we'd have it +# doubled here as well, as: `/host//opt/kata` +host_install_dir="/host${dest_dir}" # If we fail for any reason a message will be displayed die() { @@ -238,6 +247,43 @@ function get_tdx_ovmf_path_from_distro() { esac } +function adjust_qemu_cmdline() { + shim="${1}" + config_path="${2}" + qemu_share="${shim}" + + # The paths on the kata-containers tarball side look like: + # ${dest_dir}/opt/kata/share/kata-qemu/qemu + # ${dest_dir}/opt/kata/share/kata-qemu-snp-experimnental/qemu + [[ "${shim}" =~ ^(qemu-snp|qemu-nvidia-snp)$ ]] && qemu_share=${shim}-experimental + + qemu_binary=$(tomlq '.hypervisor.qemu.path' ${config_path} | tr -d \") + qemu_binary_script="${qemu_binary}-installation-prefix" + qemu_binary_script_host_path="/host/${qemu_binary_script}" + + if [[ ! -f ${qemu_binary_script_host_path} ]]; then + # From the QEMU man page: + # ``` + # -L path + # Set the directory for the BIOS, VGA BIOS and keymaps. + # To list all the data directories, use -L help. + # ``` + # + # The reason we have to do this here, is because otherwise QEMU + # will only look for those files in specific paths, which are + # tied to the location of the PREFIX used during build time + # (/opt/kata, in our case). + cat <${qemu_binary_script_host_path} +#!/usr/bin/env bash + +exec ${qemu_binary} "\$@" -L ${dest_dir}/share/kata-${qemu_share}/qemu/ +EOF + chmod +x ${qemu_binary_script_host_path} + fi + + sed -i -e "s|${qemu_binary}|${qemu_binary_script}|" ${config_path} +} + function install_artifacts() { echo "copying kata artifacts onto host" @@ -303,6 +349,16 @@ function install_artifacts() { ;; esac fi + + if [ -n "${INSTALLATION_PREFIX}" ]; then + # We could always do this sed, regardless, but I have a strong preference + # on not touching the configuration files unless extremelly needed + sed -i -e "s|${default_dest_dir}|${dest_dir}|g" "${kata_config_file}" + + # Let's only adjust qemu_cmdline for the QEMUs that we build and ship ourselves + [[ "${shim}" =~ ^(qemu|qemu-snp|qemu-nvidia-gpu|qemu-nvidia-gpu-snp|qemu-sev|qemu-se)$ ]] && \ + adjust_qemu_cmdline "${shim}" "${kata_config_file}" + fi done # Allow Mariner to use custom configuration. @@ -602,6 +658,7 @@ function main() { echo "* AGENT_HTTPS_PROXY: ${AGENT_HTTPS_PROXY}" echo "* AGENT_NO_PROXY: ${AGENT_NO_PROXY}" echo "* PULL_TYPE_MAPPING: ${PULL_TYPE_MAPPING}" + echo "* INSTALLATION_PREFIX: ${INSTALLATION_PREFIX}" # script requires that user is root euid=$(id -u)