Merge pull request #8483 from fidencio/topic/move-rust-config-files-to-subdir-based-on-jodh-approach

build/kata-deploy: Move rust runtime config files to runtime-rs directory -- based on #8445

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -27,8 +27,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        rust-runtime:
-          - false
         host_os:
           - ubuntu
         vmm:
@@ -42,8 +40,6 @@ jobs:
         include:
           - host_os: cbl-mariner
             vmm: clh
-          - dragonball:
-            rust-runtime: true
     runs-on: ubuntu-latest
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}
@@ -55,7 +51,6 @@ jobs:
       KUBERNETES: "vanilla"
       USING_NFD: "false"
       K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }}
-      RUST_RUNTIME: ${{ matrix.rust-runtime }}
     steps:
       - uses: actions/checkout@v4
         with:

src/libs/kata-types/src/config/default.rs

@@ -14,9 +14,10 @@ use lazy_static::lazy_static;
 lazy_static! {
     /// Default configuration file paths, vendor may extend the list
     pub static ref DEFAULT_RUNTIME_CONFIGURATIONS: Vec::<&'static str> = vec![
-        "/etc/kata-containers/configuration.toml",
+        // The rust runtime specific paths
-        "/usr/share/defaults/kata-containers/configuration.toml",
+        "/etc/kata-containers/runtime-rs/configuration.toml",
-        "/opt/kata/share/defaults/kata-containers/configuration.toml",
+        "/usr/share/defaults/kata-containers/runtime-rs/configuration.toml",
+        "/opt/kata/share/defaults/kata-containers/runtime-rs/configuration.toml",
     ];
 }
 

src/runtime-rs/Makefile

@@ -189,7 +189,7 @@ KNOWN_HYPERVISORS =
 # List of hypervisors known for the current architecture
 KNOWN_HYPERVISORS =
 
-CONFDIR := $(DEFAULTSDIR)/$(PROJECT_DIR)
+CONFDIR := $(DEFAULTSDIR)/$(PROJECT_DIR)/runtime-rs
 SYSCONFDIR := $(SYSCONFDIR)/$(PROJECT_DIR)
 ##VAR CONFIG_PATH=<path> Main configuration file location for stateless systems
 CONFIG_PATH := $(abspath $(CONFDIR)/$(CONFIG_FILE))

tests/common.bash

@@ -80,23 +80,56 @@ function is_a_kata_runtime() {
 # Gets versions and paths of all the components
 # list in kata-env
 function extract_kata_env() {
-	RUNTIME_CONFIG_PATH=$(kata-runtime kata-env --json | jq -r .Runtime.Config.Path)
+	local cmd
-	RUNTIME_VERSION=$(kata-runtime kata-env --json | jq -r .Runtime.Version | grep Semver | cut -d'"' -f4)
+	local config_path
-	RUNTIME_COMMIT=$(kata-runtime kata-env --json | jq -r .Runtime.Version | grep Commit | cut -d'"' -f4)
+	local runtime_version
-	RUNTIME_PATH=$(kata-runtime kata-env --json | jq -r .Runtime.Path)
+	local runtime_version_semver
+	local runtime_version_commit
+	local runtime_path
+	local hypervisor_path
+	local virtiofsd_path
+	local initrd_path
+	case "${KATA_HYPERVISOR}" in
+		dragonball)
+			cmd=kata-ctl
+			config_path=".runtime.config.path"
+			runtime_version=".runtime.version"
+			runtime_version_semver="semver"
+			runtime_version_commit="commit"
+			runtime_path=".runtime.path"
+			hypervisor_path=".hypervisor.path"
+			virtio_fs_daemon_path=".hypervisor.virtio_fs_daemon"
+			initrd_path=".initrd.path"
+			;;
+		*)
+			cmd=kata-runtime
+			config_path=".Runtime.Config.Path"
+			runtime_version=".Runtime.Version"
+			runtime_version_semver="Semver"
+			runtime_version_commit="Commit"
+			runtime_path=".Runtime.Path"
+			hypervisor_path=".Hypervisor.Path"
+			virtio_fs_daemon_path=".Hypervisor.VirtioFSDaemon"
+			initrd_path=".Initrd.Path"
+			;;
+	esac
+	RUNTIME_CONFIG_PATH=$(sudo ${cmd} env --json | jq -r ${config_path})
+	RUNTIME_VERSION=$(sudo ${cmd} env --json | jq -r ${runtime_version} | grep ${runtime_version_semver} | cut -d'"' -f4)
+	RUNTIME_COMMIT=$(sudo ${cmd} env --json | jq -r ${runtime_version} | grep ${runtime_version_commit} | cut -d'"' -f4)
+	RUNTIME_PATH=$(sudo ${cmd} env --json | jq -r ${runtime_path})
 
 	# Shimv2 path is being affected by https://github.com/kata-containers/kata-containers/issues/1151
 	SHIM_PATH=$(readlink $(command -v containerd-shim-kata-v2))
 	SHIM_VERSION=${RUNTIME_VERSION}
 
-	HYPERVISOR_PATH=$(kata-runtime kata-env --json | jq -r .Hypervisor.Path)
+	HYPERVISOR_PATH=$(sudo ${cmd} env --json | jq -r ${hypervisor_path})
-	# TODO: there is no kata-runtime of rust version currently
+	# TODO: there is no ${cmd} of rust version currently
 	if [ "${KATA_HYPERVISOR}" != "dragonball" ]; then
 		HYPERVISOR_VERSION=$(sudo -E ${HYPERVISOR_PATH} --version | head -n1)
 	fi
-	VIRTIOFSD_PATH=$(kata-runtime kata-env --json | jq -r .Hypervisor.VirtioFSDaemon)
+	VIRTIOFSD_PATH=$(sudo ${cmd} env --json | jq -r ${virtio_fs_daemon_path})
 
-	INITRD_PATH=$(kata-runtime kata-env --json | jq -r .Initrd.Path)
+	INITRD_PATH=$(sudo ${cmd} env --json | jq -r ${initrd_path})
 }
 
 # Checks that processes are not running
@@ -105,8 +138,8 @@ function check_processes() {
 
 	# Only check the kata-env if we have managed to find the kata executable...
 	if [ -x "$RUNTIME_PATH" ]; then
-		local vsock_configured=$($RUNTIME_PATH kata-env | awk '/UseVSock/ {print $3}')
+		local vsock_configured=$($RUNTIME_PATH env | awk '/UseVSock/ {print $3}')
-		local vsock_supported=$($RUNTIME_PATH kata-env | awk '/SupportVSock/ {print $3}')
+		local vsock_supported=$($RUNTIME_PATH env | awk '/SupportVSock/ {print $3}')
 	else
 		local vsock_configured="false"
 		local vsock_supported="false"
@@ -366,16 +399,21 @@ function install_kata() {
 # points to the hypervisor passed by KATA_HYPERVISOR env var.
 function enabling_hypervisor() {
 	declare -r KATA_DIR="/opt/kata"
-	declare -r CONFIG_DIR="${KATA_DIR}/share/defaults/kata-containers"
-	declare -r SRC_HYPERVISOR_CONFIG="${CONFIG_DIR}/configuration-${KATA_HYPERVISOR}.toml"
-	declare -r DEST_KATA_CONFIG="${CONFIG_DIR}/configuration.toml"
 	declare -r CONTAINERD_SHIM_KATA="/usr/local/bin/containerd-shim-kata-${KATA_HYPERVISOR}-v2"
 
-	if [[ ${KATA_HYPERVISOR} == "dragonball" ]]; then
+	case "${KATA_HYPERVISOR}" in
+		dragonball)
 			sudo ln -sf "${KATA_DIR}/runtime-rs/bin/containerd-shim-kata-v2" "${CONTAINERD_SHIM_KATA}"
-	else
+			declare -r CONFIG_DIR="${KATA_DIR}/share/defaults/kata-containers/runtime-rs"
+			;;
+		*)
 			sudo ln -sf "${KATA_DIR}/bin/containerd-shim-kata-v2" "${CONTAINERD_SHIM_KATA}"
-	fi
+			declare -r CONFIG_DIR="${KATA_DIR}/share/defaults/kata-containers"
+			;;
+	esac
+
+	declare -r SRC_HYPERVISOR_CONFIG="${CONFIG_DIR}/configuration-${KATA_HYPERVISOR}.toml"
+	declare -r DEST_KATA_CONFIG="${CONFIG_DIR}/configuration.toml"
 
 	sudo ln -sf "${SRC_HYPERVISOR_CONFIG}" "${DEST_KATA_CONFIG}"
 }

tests/integration/nydus/nydus_tests.sh

@@ -21,7 +21,7 @@ kata_config_backup="/tmp/kata-configuration.toml"
 SYSCONFIG_FILE="/etc/kata-containers/configuration.toml"
 DEFAULT_CONFIG_FILE="/opt/kata/share/defaults/kata-containers/configuration-qemu.toml"
 CLH_CONFIG_FILE="/opt/kata/share/defaults/kata-containers/configuration-clh.toml"
-DB_CONFIG_FILE="/opt/kata/share/defaults/kata-containers/configuration-dragonball.toml"
+DB_CONFIG_FILE="/opt/kata/share/defaults/kata-containers/runtime-rs/configuration-dragonball.toml"
 need_restore_containerd_config=false
 containerd_config="/etc/containerd/config.toml"
 containerd_config_backup="/tmp/containerd.config.toml"
@@ -34,6 +34,14 @@ if [ "$KATA_HYPERVISOR" != "qemu" ] && [ "$KATA_HYPERVISOR" != "clh" ] && [ "$KA
 	exit 0
 fi
 
+case "$KATA_HYPERVISOR" in
+	dragonball)
+		SYSCONFIG_FILE="/etc/kata-containers/runtime-rs/configuration.toml"
+		;;
+	*)
+		;;
+esac
+
 function setup_nydus() {
 	# Config nydus snapshotter
 	sudo -E cp "$dir_path/nydusd-config.json" /etc/
@@ -46,7 +54,7 @@ function setup_nydus() {
 }
 
 function config_kata() {
-	sudo mkdir -p /etc/kata-containers
+	sudo mkdir -p $(dirname $SYSCONFIG_FILE)
 	if [ -f "$SYSCONFIG_FILE" ]; then
 		need_restore_kata_config=true
 		sudo cp -a "${SYSCONFIG_FILE}" "${kata_config_backup}"

tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml

@@ -30,7 +30,7 @@ spec:
             - name: DEBUG
               value: "false"
             - name: SHIMS
-              value: "clh dragonball fc qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx qemu remote stratovirt"
+              value: "clh cloud-hypervisor dragonball fc qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx qemu stratovirt"
             - name: DEFAULT_SHIM
               value: "qemu"
             - name: CREATE_RUNTIMECLASSES

tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml

@@ -32,7 +32,7 @@ spec:
             - name: DEBUG
               value: "false"
             - name: SHIMS
-              value: "clh dragonball fc qemu qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx"
+              value: "clh cloud-hypervisor dragonball fc qemu qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx"
             - name: DEFAULT_SHIM
               value: "qemu"
             - name: CREATE_RUNTIMECLASSES

tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml

@@ -32,7 +32,7 @@ spec:
             - name: DEBUG
               value: "false"
             - name: SHIMS
-              value: "clh dragonball fc qemu qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx remote stratovirt"
+              value: "clh cloud-hypervisor dragonball fc qemu qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx stratovirt"
             - name: DEFAULT_SHIM
               value: "qemu"
             - name: CREATE_RUNTIMECLASSES

tools/packaging/kata-deploy/runtimeclasses/kata-cloud-hypervisor.yaml

@@ -0,0 +1,13 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-cloud-hypervisor
+handler: kata-cloud-hypervisor
+overhead:
+    podFixed:
+        memory: "130Mi"
+        cpu: "250m"
+scheduling:
+  nodeSelector:
+    katacontainers.io/kata-runtime: "true"

tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml

@@ -14,6 +14,19 @@ scheduling:
 ---
 kind: RuntimeClass
 apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-cloud-hypervisor
+handler: kata-cloud-hypervisor
+overhead:
+    podFixed:
+        memory: "130Mi"
+        cpu: "250m"
+scheduling:
+  nodeSelector:
+    katacontainers.io/kata-runtime: "true"
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
 metadata:
     name: kata-dragonball
 handler: kata-dragonball

tools/packaging/kata-deploy/scripts/kata-deploy.sh

@@ -105,6 +105,42 @@ function get_container_runtime() {
 	fi
 }
 
+function get_kata_containers_config_path() {
+	local shim="$1"
+
+	# Directory holding pristine configuration files for the current default golang runtime.
+	local golang_config_path="/opt/kata/share/defaults/kata-containers/"
+
+	# Directory holding pristine configuration files for the new rust runtime.
+	#
+	# These are put into a separate directory since:
+	#
+	# - In some cases, the rust runtime configuration syntax is
+	#   slightly different to the golang runtime configuration files
+	#   so some hypervisors need two different configuration files,
+	#   one for reach runtime type (for example Cloud Hypervisor which
+	#   uses 'clh' for the golang runtime and 'cloud-hypervisor' for
+	#   the rust runtime.
+	#
+	# - Some hypervisors only currently work with the golang runtime.
+	#
+	# - Some hypervisors only work with the rust runtime (dragonball).
+	#
+	# See: https://github.com/kata-containers/kata-containers/issues/6020
+	local rust_config_path="${golang_config_path}/runtime-rs"
+
+	local config_path
+
+	# Map the runtime shim name to the appropriate configuration
+	# file directory.
+	case "$shim" in
+		cloud-hypervisor | dragonball) config_path="$rust_config_path" ;;
+		*) config_path="$golang_config_path" ;;
+	esac
+
+	echo "$config_path"
+}
+
 function install_artifacts() {
 	echo "copying kata artifacts onto host"
 	cp -au /opt/kata-artifacts/opt/kata/* /opt/kata/
@@ -112,8 +148,12 @@ function install_artifacts() {
 	[ -d /opt/kata/runtime-rs/bin ] && \
 		chmod +x /opt/kata/runtime-rs/bin/*
 
-	config_path="/opt/kata/share/defaults/kata-containers/"
+	local config_path
+
 	for shim in "${shims[@]}"; do
+		config_path=$(get_kata_containers_config_path "${shim}")
+		mkdir -p "$config_path"
+
 		local kata_config_file="${config_path}/configuration-${shim}.toml"
 		# Allow enabling debug for Kata Containers
 		if [[ "${DEBUG}" == "true" ]]; then
@@ -204,11 +244,15 @@ function configure_different_shims_base() {
 
 		backup_shim "${shim_file}"
 
-		if [[ "${shim}" == "dragonball" ]]; then
+		# Map the runtime shim name to the appropriate
-			ln -sf /opt/kata/runtime-rs/bin/containerd-shim-kata-v2 "${shim_file}"
+		# containerd-shim-kata-v2 binary
-		else
+		case "$shim" in
-			ln -sf /opt/kata/bin/containerd-shim-kata-v2 "${shim_file}"
+			cloud-hypervisor | dragonball)
-		fi
+				ln -sf /opt/kata/runtime-rs/bin/containerd-shim-kata-v2 "${shim_file}" ;;
+			*)
+				ln -sf /opt/kata/bin/containerd-shim-kata-v2 "${shim_file}" ;;
+		esac
+
 		chmod +x "$shim_file"
 
 		if [ "${shim}" == "${default_shim}" ]; then
@@ -257,9 +301,11 @@ function configure_crio_runtime() {
 		configuration+="-$1"
 	fi
 
+	local config_path=$(get_kata_containers_config_path "${1}")
+
 	local kata_path="/usr/local/bin/containerd-shim-${runtime}-v2"
 	local kata_conf="crio.runtime.runtimes.${runtime}"
-	local kata_config_path="/opt/kata/share/defaults/kata-containers/$configuration.toml"
+	local kata_config_path="${config_path}/${configuration}.toml"
 
 	cat <<EOF | tee -a "$crio_drop_in_conf_file"
 
@@ -314,7 +360,7 @@ function configure_containerd_runtime() {
 	local runtime_table="plugins.${pluginid}.containerd.runtimes.$runtime"
 	local runtime_type="io.containerd.$runtime.v2"
 	local options_table="$runtime_table.options"
-	local config_path="/opt/kata/share/defaults/kata-containers/$configuration.toml"
+	local config_path="$(get_kata_containers_config_path "$2")/$configuration.toml"
 	if grep -q "\[$runtime_table\]" $containerd_conf_file; then
 		echo "Configuration exists for $runtime_table, overwriting"
 		sed -i "/\[$runtime_table\]/,+1s#runtime_type.*#runtime_type = \"${runtime_type}\"#" $containerd_conf_file