diff --git a/tests/integration/cri-containerd/integration-tests.sh b/tests/integration/cri-containerd/integration-tests.sh index 7a144cea34..75fafd66bb 100755 --- a/tests/integration/cri-containerd/integration-tests.sh +++ b/tests/integration/cri-containerd/integration-tests.sh @@ -40,7 +40,9 @@ containerd_shim_path="$(command -v containerd-shim || true)" tmp_dir=$(mktemp -t -d test-cri-containerd.XXXX) readonly tmp_dir export REPORT_DIR="${tmp_dir}" -readonly CONTAINERD_CONFIG_FILE="${tmp_dir}/test-containerd-config" +readonly CONTAINERD_CONFIG_FILE="${tmp_dir}/test-containerd-config.toml" +readonly CONTAINERD_CONF_D="${tmp_dir}/conf.d" +readonly CONTAINERD_SYSTEM_FRAGMENT_PREFIX="${CONTAINERD_SYSTEM_FRAGMENT_PREFIX:-ZZ-cri-integration-test-}" readonly CONTAINERD_CONFIG_FILE_TEMP="${CONTAINERD_CONFIG_FILE}.temp" readonly default_containerd_config="/etc/containerd/config.toml" readonly default_containerd_config_backup="${CONTAINERD_CONFIG_FILE}.backup" @@ -81,6 +83,7 @@ function ci_cleanup() { if [[ -e "${default_containerd_config_backup}" ]]; then echo "restore containerd config" sudo systemctl stop containerd + sudo rm -f /etc/containerd/conf.d/"${CONTAINERD_SYSTEM_FRAGMENT_PREFIX}"*.toml sudo cp "${default_containerd_config_backup}" "${default_containerd_config}" fi @@ -112,17 +115,27 @@ function create_containerd_config() { runtime_binary_path="" fi - # check containerd config version - if containerd config default | grep -qE "^version = [34]"; then - pluginid=\"io.containerd.cri.v1.runtime\" - else - pluginid="cri" - fi + local cfg_schema + cfg_schema="$(_containerd_blob_schema_version "$(PATH="${PATH}:/usr/local/bin:/usr/local/sbin" containerd config default 2>/dev/null || true)")" + [[ "${cfg_schema}" =~ ^[0-9]+$ ]] || die "could not read containerd config schema from config default" + info "Kata Config Path ${runtime_config_path}, Runtime Binary Name ${runtime_binary_path}" -cat << EOF | sudo tee "${CONTAINERD_CONFIG_FILE}" + if [[ "${cfg_schema}" -ge 3 ]]; then + # containerd v2.x (schema v3+): base config (imports conf.d) plus drop-in + # fragments for sockets, debug, runtime and the linux shim. + local pluginid=\"io.containerd.cri.v1.runtime\" + + mkdir -p "${CONTAINERD_CONF_D}" + containerd_render_config_default_with_imports "${CONTAINERD_CONFIG_FILE}" "${CONTAINERD_CONF_D}" + containerd_emit_rootful_api_socket_overrides "${cfg_schema}" > "${CONTAINERD_CONF_D}/10-cri-test-api-sockets.toml" + + cat <<'EOF' > "${CONTAINERD_CONF_D}/20-cri-test-debug.toml" [debug] level = "debug" +EOF + + cat << EOF > "${CONTAINERD_CONF_D}/30-cri-test-runtime.toml" [plugins] [plugins.${pluginid}] [plugins.${pluginid}.containerd] @@ -137,11 +150,83 @@ cat << EOF | sudo tee "${CONTAINERD_CONFIG_FILE}" [plugins.${pluginid}.containerd.runtimes.${runtime}.options] ConfigPath = "${runtime_config_path}" BinaryName = "${runtime_binary_path}" -$( [[ -n "${containerd_shim_path}" ]] && \ -echo "[plugins.linux]" && \ -echo " shim = \"${containerd_shim_path}\"" -) EOF + + if [[ -n "${containerd_shim_path}" ]]; then + cat << EOF > "${CONTAINERD_CONF_D}/40-cri-test-linux.toml" +[plugins.linux] + shim = "${containerd_shim_path}" +EOF + else + rm -f "${CONTAINERD_CONF_D}/40-cri-test-linux.toml" + fi + else + # containerd v1.x (schema v2): conf.d is not honoured the same way, so + # write a single complete, self-contained config file. The v1.x default + # API sockets are already root-owned, so no socket override is needed. + local pluginid=\"io.containerd.grpc.v1.cri\" + local linux_shim_block="" + if [[ -n "${containerd_shim_path}" ]]; then + linux_shim_block="$(printf '\n[plugins.linux]\n shim = "%s"\n' "${containerd_shim_path}")" + fi + + mkdir -p "${CONTAINERD_CONF_D}" + cat << EOF > "${CONTAINERD_CONFIG_FILE}" +version = 2 + +[debug] + level = "debug" + +[plugins] + [plugins.${pluginid}] + [plugins.${pluginid}.containerd] + default_runtime_name = "${runtime}" + [plugins.${pluginid}.containerd.runtimes.${runtime}] + runtime_type = "${runtime_type}" + $( [[ "${kata_annotations}" -eq 1 ]] && \ + echo 'pod_annotations = ["io.katacontainers.*"]' && \ + echo ' container_annotations = ["io.katacontainers.*"]' + ) + [plugins.${pluginid}.containerd.runtimes.${runtime}.options] + ConfigPath = "${runtime_config_path}" + BinaryName = "${runtime_binary_path}" +${linux_shim_block} +EOF + fi +} + +# Push the test containerd config to the host paths used by crictl. +function deploy_cri_integration_containerd_host_config() { + sudo cp "${default_containerd_config}" "${default_containerd_config_backup}" + + local cfg_schema + cfg_schema="$(_containerd_blob_schema_version "$(PATH="${PATH}:/usr/local/bin:/usr/local/sbin" containerd config default 2>/dev/null || true)")" + + if [[ "${cfg_schema}" =~ ^[0-9]+$ ]] && [[ "${cfg_schema}" -ge 3 ]]; then + # containerd v2.x (schema v3+): base config (imports conf.d) + drop-ins. + local host_main_fragment="${REPORT_DIR}/containerd-main-for-host.toml" + + containerd_render_config_default_with_imports "${host_main_fragment}" "/etc/containerd/conf.d" + sudo mkdir -p /etc/containerd/conf.d + sudo cp "${host_main_fragment}" "${default_containerd_config}" + + shopt -s nullglob + for f in "${CONTAINERD_CONF_D}"/*.toml; do + sudo cp "${f}" "/etc/containerd/conf.d/${CONTAINERD_SYSTEM_FRAGMENT_PREFIX}$(basename "${f}")" + done + shopt -u nullglob + else + # containerd v1.x (schema v2): deploy the complete, self-contained config. + sudo cp "${CONTAINERD_CONFIG_FILE}" "${default_containerd_config}" + fi + + restart_containerd_service +} + +function restore_cri_integration_containerd_host_config() { + sudo rm -f /etc/containerd/conf.d/"${CONTAINERD_SYSTEM_FRAGMENT_PREFIX}"*.toml + sudo cp "${default_containerd_config_backup}" "${default_containerd_config}" + restart_containerd_service } function cleanup() { @@ -222,10 +307,7 @@ command: EOF fi - sudo cp "${default_containerd_config}" "${default_containerd_config_backup}" - sudo cp "${CONTAINERD_CONFIG_FILE}" "${default_containerd_config}" - - restart_containerd_service + deploy_cri_integration_containerd_host_config sudo crictl pull "${image}" podid=$(sudo crictl --timeout=5s runp "${pod_yaml}") @@ -241,8 +323,7 @@ function testContainerStop() { info "remove pod ${podid}" sudo crictl --timeout=20s rmp "${podid}" - sudo cp "${default_containerd_config_backup}" "${default_containerd_config}" - restart_containerd_service + restore_cri_integration_containerd_host_config } function TestKilledVmmCleanup() { @@ -559,10 +640,7 @@ devices: permissions: rwm EOF - sudo cp "${default_containerd_config}" "${default_containerd_config_backup}" - sudo cp "${CONTAINERD_CONFIG_FILE}" "${default_containerd_config}" - - restart_containerd_service + deploy_cri_integration_containerd_host_config sudo crictl pull "${image}" podid=$(sudo crictl --timeout=5s runp "${pod_yaml}") @@ -580,8 +658,7 @@ function stopDeviceCgroupContainers() { info "remove pod ${podid}" sudo crictl --timeout=20s rmp "${podid}" - sudo cp "${default_containerd_config_backup}" "${default_containerd_config}" - restart_containerd_service + restore_cri_integration_containerd_host_config } function TestDeviceCgroup() { @@ -673,7 +750,7 @@ function main() { export_go_toolchain_for_containerd_source_builds # Make sure the right artifacts are going to be built - sudo make clean + sudo -E make clean # the latest containerd had an issue for its e2e test, thus we should do the following # fix to workaround this issue. For much info about this issue, please see: