diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index e275869796..aea0a02406 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -27,8 +27,6 @@ jobs: strategy: fail-fast: false matrix: - rust-runtime: - - false host_os: - ubuntu vmm: @@ -42,8 +40,6 @@ jobs: include: - host_os: cbl-mariner vmm: clh - - dragonball: - rust-runtime: true runs-on: ubuntu-latest env: DOCKER_REGISTRY: ${{ inputs.registry }} @@ -55,7 +51,6 @@ jobs: KUBERNETES: "vanilla" USING_NFD: "false" K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }} - RUST_RUNTIME: ${{ matrix.rust-runtime }} steps: - uses: actions/checkout@v4 with: diff --git a/src/libs/kata-types/src/config/default.rs b/src/libs/kata-types/src/config/default.rs index f55f597da2..73f368e4f1 100644 --- a/src/libs/kata-types/src/config/default.rs +++ b/src/libs/kata-types/src/config/default.rs @@ -14,9 +14,10 @@ use lazy_static::lazy_static; lazy_static! { /// Default configuration file paths, vendor may extend the list pub static ref DEFAULT_RUNTIME_CONFIGURATIONS: Vec::<&'static str> = vec![ - "/etc/kata-containers/configuration.toml", - "/usr/share/defaults/kata-containers/configuration.toml", - "/opt/kata/share/defaults/kata-containers/configuration.toml", + // The rust runtime specific paths + "/etc/kata-containers/runtime-rs/configuration.toml", + "/usr/share/defaults/kata-containers/runtime-rs/configuration.toml", + "/opt/kata/share/defaults/kata-containers/runtime-rs/configuration.toml", ]; } diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index f492d1d334..ddf857a54b 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -189,7 +189,7 @@ KNOWN_HYPERVISORS = # List of hypervisors known for the current architecture KNOWN_HYPERVISORS = -CONFDIR := $(DEFAULTSDIR)/$(PROJECT_DIR) +CONFDIR := $(DEFAULTSDIR)/$(PROJECT_DIR)/runtime-rs SYSCONFDIR := $(SYSCONFDIR)/$(PROJECT_DIR) ##VAR CONFIG_PATH= Main configuration file location for stateless systems CONFIG_PATH := $(abspath $(CONFDIR)/$(CONFIG_FILE)) diff --git a/tests/common.bash b/tests/common.bash index b7ac98c499..e941792d4c 100644 --- a/tests/common.bash +++ b/tests/common.bash @@ -80,23 +80,56 @@ function is_a_kata_runtime() { # Gets versions and paths of all the components # list in kata-env function extract_kata_env() { - RUNTIME_CONFIG_PATH=$(kata-runtime kata-env --json | jq -r .Runtime.Config.Path) - RUNTIME_VERSION=$(kata-runtime kata-env --json | jq -r .Runtime.Version | grep Semver | cut -d'"' -f4) - RUNTIME_COMMIT=$(kata-runtime kata-env --json | jq -r .Runtime.Version | grep Commit | cut -d'"' -f4) - RUNTIME_PATH=$(kata-runtime kata-env --json | jq -r .Runtime.Path) + local cmd + local config_path + local runtime_version + local runtime_version_semver + local runtime_version_commit + local runtime_path + local hypervisor_path + local virtiofsd_path + local initrd_path + case "${KATA_HYPERVISOR}" in + dragonball) + cmd=kata-ctl + config_path=".runtime.config.path" + runtime_version=".runtime.version" + runtime_version_semver="semver" + runtime_version_commit="commit" + runtime_path=".runtime.path" + hypervisor_path=".hypervisor.path" + virtio_fs_daemon_path=".hypervisor.virtio_fs_daemon" + initrd_path=".initrd.path" + ;; + *) + cmd=kata-runtime + config_path=".Runtime.Config.Path" + runtime_version=".Runtime.Version" + runtime_version_semver="Semver" + runtime_version_commit="Commit" + runtime_path=".Runtime.Path" + hypervisor_path=".Hypervisor.Path" + virtio_fs_daemon_path=".Hypervisor.VirtioFSDaemon" + initrd_path=".Initrd.Path" + ;; + esac + RUNTIME_CONFIG_PATH=$(sudo ${cmd} env --json | jq -r ${config_path}) + RUNTIME_VERSION=$(sudo ${cmd} env --json | jq -r ${runtime_version} | grep ${runtime_version_semver} | cut -d'"' -f4) + RUNTIME_COMMIT=$(sudo ${cmd} env --json | jq -r ${runtime_version} | grep ${runtime_version_commit} | cut -d'"' -f4) + RUNTIME_PATH=$(sudo ${cmd} env --json | jq -r ${runtime_path}) # Shimv2 path is being affected by https://github.com/kata-containers/kata-containers/issues/1151 SHIM_PATH=$(readlink $(command -v containerd-shim-kata-v2)) SHIM_VERSION=${RUNTIME_VERSION} - HYPERVISOR_PATH=$(kata-runtime kata-env --json | jq -r .Hypervisor.Path) - # TODO: there is no kata-runtime of rust version currently + HYPERVISOR_PATH=$(sudo ${cmd} env --json | jq -r ${hypervisor_path}) + # TODO: there is no ${cmd} of rust version currently if [ "${KATA_HYPERVISOR}" != "dragonball" ]; then HYPERVISOR_VERSION=$(sudo -E ${HYPERVISOR_PATH} --version | head -n1) fi - VIRTIOFSD_PATH=$(kata-runtime kata-env --json | jq -r .Hypervisor.VirtioFSDaemon) + VIRTIOFSD_PATH=$(sudo ${cmd} env --json | jq -r ${virtio_fs_daemon_path}) - INITRD_PATH=$(kata-runtime kata-env --json | jq -r .Initrd.Path) + INITRD_PATH=$(sudo ${cmd} env --json | jq -r ${initrd_path}) } # Checks that processes are not running @@ -105,8 +138,8 @@ function check_processes() { # Only check the kata-env if we have managed to find the kata executable... if [ -x "$RUNTIME_PATH" ]; then - local vsock_configured=$($RUNTIME_PATH kata-env | awk '/UseVSock/ {print $3}') - local vsock_supported=$($RUNTIME_PATH kata-env | awk '/SupportVSock/ {print $3}') + local vsock_configured=$($RUNTIME_PATH env | awk '/UseVSock/ {print $3}') + local vsock_supported=$($RUNTIME_PATH env | awk '/SupportVSock/ {print $3}') else local vsock_configured="false" local vsock_supported="false" @@ -366,16 +399,21 @@ function install_kata() { # points to the hypervisor passed by KATA_HYPERVISOR env var. function enabling_hypervisor() { declare -r KATA_DIR="/opt/kata" - declare -r CONFIG_DIR="${KATA_DIR}/share/defaults/kata-containers" - declare -r SRC_HYPERVISOR_CONFIG="${CONFIG_DIR}/configuration-${KATA_HYPERVISOR}.toml" - declare -r DEST_KATA_CONFIG="${CONFIG_DIR}/configuration.toml" declare -r CONTAINERD_SHIM_KATA="/usr/local/bin/containerd-shim-kata-${KATA_HYPERVISOR}-v2" - if [[ ${KATA_HYPERVISOR} == "dragonball" ]]; then - sudo ln -sf "${KATA_DIR}/runtime-rs/bin/containerd-shim-kata-v2" "${CONTAINERD_SHIM_KATA}" - else - sudo ln -sf "${KATA_DIR}/bin/containerd-shim-kata-v2" "${CONTAINERD_SHIM_KATA}" - fi + case "${KATA_HYPERVISOR}" in + dragonball) + sudo ln -sf "${KATA_DIR}/runtime-rs/bin/containerd-shim-kata-v2" "${CONTAINERD_SHIM_KATA}" + declare -r CONFIG_DIR="${KATA_DIR}/share/defaults/kata-containers/runtime-rs" + ;; + *) + sudo ln -sf "${KATA_DIR}/bin/containerd-shim-kata-v2" "${CONTAINERD_SHIM_KATA}" + declare -r CONFIG_DIR="${KATA_DIR}/share/defaults/kata-containers" + ;; + esac + + declare -r SRC_HYPERVISOR_CONFIG="${CONFIG_DIR}/configuration-${KATA_HYPERVISOR}.toml" + declare -r DEST_KATA_CONFIG="${CONFIG_DIR}/configuration.toml" sudo ln -sf "${SRC_HYPERVISOR_CONFIG}" "${DEST_KATA_CONFIG}" } diff --git a/tests/integration/nydus/nydus_tests.sh b/tests/integration/nydus/nydus_tests.sh index 9e3f655b9c..41a4cbbfc5 100755 --- a/tests/integration/nydus/nydus_tests.sh +++ b/tests/integration/nydus/nydus_tests.sh @@ -21,7 +21,7 @@ kata_config_backup="/tmp/kata-configuration.toml" SYSCONFIG_FILE="/etc/kata-containers/configuration.toml" DEFAULT_CONFIG_FILE="/opt/kata/share/defaults/kata-containers/configuration-qemu.toml" CLH_CONFIG_FILE="/opt/kata/share/defaults/kata-containers/configuration-clh.toml" -DB_CONFIG_FILE="/opt/kata/share/defaults/kata-containers/configuration-dragonball.toml" +DB_CONFIG_FILE="/opt/kata/share/defaults/kata-containers/runtime-rs/configuration-dragonball.toml" need_restore_containerd_config=false containerd_config="/etc/containerd/config.toml" containerd_config_backup="/tmp/containerd.config.toml" @@ -34,6 +34,14 @@ if [ "$KATA_HYPERVISOR" != "qemu" ] && [ "$KATA_HYPERVISOR" != "clh" ] && [ "$KA exit 0 fi +case "$KATA_HYPERVISOR" in + dragonball) + SYSCONFIG_FILE="/etc/kata-containers/runtime-rs/configuration.toml" + ;; + *) + ;; +esac + function setup_nydus() { # Config nydus snapshotter sudo -E cp "$dir_path/nydusd-config.json" /etc/ @@ -46,7 +54,7 @@ function setup_nydus() { } function config_kata() { - sudo mkdir -p /etc/kata-containers + sudo mkdir -p $(dirname $SYSCONFIG_FILE) if [ -f "$SYSCONFIG_FILE" ]; then need_restore_kata_config=true sudo cp -a "${SYSCONFIG_FILE}" "${kata_config_backup}" diff --git a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml index df7b715a9e..7239cdf8b5 100644 --- a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml +++ b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml @@ -30,7 +30,7 @@ spec: - name: DEBUG value: "false" - name: SHIMS - value: "clh dragonball fc qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx qemu remote stratovirt" + value: "clh cloud-hypervisor dragonball fc qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx qemu stratovirt" - name: DEFAULT_SHIM value: "qemu" - name: CREATE_RUNTIMECLASSES diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml index aa466cfc6f..cf78b8e5c7 100644 --- a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml +++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml @@ -32,7 +32,7 @@ spec: - name: DEBUG value: "false" - name: SHIMS - value: "clh dragonball fc qemu qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx" + value: "clh cloud-hypervisor dragonball fc qemu qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx" - name: DEFAULT_SHIM value: "qemu" - name: CREATE_RUNTIMECLASSES diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml index 03cbe4a53e..e746c6f32b 100644 --- a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml +++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml @@ -32,7 +32,7 @@ spec: - name: DEBUG value: "false" - name: SHIMS - value: "clh dragonball fc qemu qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx remote stratovirt" + value: "clh cloud-hypervisor dragonball fc qemu qemu-nvidia-gpu qemu-sev qemu-snp qemu-tdx stratovirt" - name: DEFAULT_SHIM value: "qemu" - name: CREATE_RUNTIMECLASSES diff --git a/tools/packaging/kata-deploy/runtimeclasses/kata-cloud-hypervisor.yaml b/tools/packaging/kata-deploy/runtimeclasses/kata-cloud-hypervisor.yaml new file mode 100644 index 0000000000..c9c16aa882 --- /dev/null +++ b/tools/packaging/kata-deploy/runtimeclasses/kata-cloud-hypervisor.yaml @@ -0,0 +1,13 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 +metadata: + name: kata-cloud-hypervisor +handler: kata-cloud-hypervisor +overhead: + podFixed: + memory: "130Mi" + cpu: "250m" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" diff --git a/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml b/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml index 8736ad6325..b96ac44f8e 100644 --- a/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml +++ b/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml @@ -14,6 +14,19 @@ scheduling: --- kind: RuntimeClass apiVersion: node.k8s.io/v1 +metadata: + name: kata-cloud-hypervisor +handler: kata-cloud-hypervisor +overhead: + podFixed: + memory: "130Mi" + cpu: "250m" +scheduling: + nodeSelector: + katacontainers.io/kata-runtime: "true" +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1 metadata: name: kata-dragonball handler: kata-dragonball diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index 178a33279c..58517480f6 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -105,6 +105,42 @@ function get_container_runtime() { fi } +function get_kata_containers_config_path() { + local shim="$1" + + # Directory holding pristine configuration files for the current default golang runtime. + local golang_config_path="/opt/kata/share/defaults/kata-containers/" + + # Directory holding pristine configuration files for the new rust runtime. + # + # These are put into a separate directory since: + # + # - In some cases, the rust runtime configuration syntax is + # slightly different to the golang runtime configuration files + # so some hypervisors need two different configuration files, + # one for reach runtime type (for example Cloud Hypervisor which + # uses 'clh' for the golang runtime and 'cloud-hypervisor' for + # the rust runtime. + # + # - Some hypervisors only currently work with the golang runtime. + # + # - Some hypervisors only work with the rust runtime (dragonball). + # + # See: https://github.com/kata-containers/kata-containers/issues/6020 + local rust_config_path="${golang_config_path}/runtime-rs" + + local config_path + + # Map the runtime shim name to the appropriate configuration + # file directory. + case "$shim" in + cloud-hypervisor | dragonball) config_path="$rust_config_path" ;; + *) config_path="$golang_config_path" ;; + esac + + echo "$config_path" +} + function install_artifacts() { echo "copying kata artifacts onto host" cp -au /opt/kata-artifacts/opt/kata/* /opt/kata/ @@ -112,8 +148,12 @@ function install_artifacts() { [ -d /opt/kata/runtime-rs/bin ] && \ chmod +x /opt/kata/runtime-rs/bin/* - config_path="/opt/kata/share/defaults/kata-containers/" + local config_path + for shim in "${shims[@]}"; do + config_path=$(get_kata_containers_config_path "${shim}") + mkdir -p "$config_path" + local kata_config_file="${config_path}/configuration-${shim}.toml" # Allow enabling debug for Kata Containers if [[ "${DEBUG}" == "true" ]]; then @@ -204,11 +244,15 @@ function configure_different_shims_base() { backup_shim "${shim_file}" - if [[ "${shim}" == "dragonball" ]]; then - ln -sf /opt/kata/runtime-rs/bin/containerd-shim-kata-v2 "${shim_file}" - else - ln -sf /opt/kata/bin/containerd-shim-kata-v2 "${shim_file}" - fi + # Map the runtime shim name to the appropriate + # containerd-shim-kata-v2 binary + case "$shim" in + cloud-hypervisor | dragonball) + ln -sf /opt/kata/runtime-rs/bin/containerd-shim-kata-v2 "${shim_file}" ;; + *) + ln -sf /opt/kata/bin/containerd-shim-kata-v2 "${shim_file}" ;; + esac + chmod +x "$shim_file" if [ "${shim}" == "${default_shim}" ]; then @@ -257,9 +301,11 @@ function configure_crio_runtime() { configuration+="-$1" fi + local config_path=$(get_kata_containers_config_path "${1}") + local kata_path="/usr/local/bin/containerd-shim-${runtime}-v2" local kata_conf="crio.runtime.runtimes.${runtime}" - local kata_config_path="/opt/kata/share/defaults/kata-containers/$configuration.toml" + local kata_config_path="${config_path}/${configuration}.toml" cat <