kata-containers/tests/integration/cri-containerd/integration-tests.sh

#!/bin/bash
#
# Copyright (c) 2017-2023 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
#

# shellcheck disable=SC2154
[[ "${DEBUG}" != "" ]] && set -o xtrace
set -o errexit
set -o nounset
set -o pipefail
set -o errtrace

SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
# shellcheck source=/dev/null
source "${SCRIPT_PATH}/../../common.bash"

# runc is installed in /usr/local/sbin/ add that path
export PATH="${PATH}:/usr/local/sbin"

# golang is installed in /usr/local/go/bin/ add that path
export PATH="${PATH}:/usr/local/go/bin"

#the latest containerd from 2.0 need to set the CGROUP_DRIVER for e2e testing
export CGROUP_DRIVER=""

# Runtime to be used for testing
KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
RUNTIME=${RUNTIME:-containerd-shim-kata-${KATA_HYPERVISOR}-v2}
FACTORY_TEST=${FACTORY_TEST:-""}
ARCH=$(uname -m)
SANDBOXER=${SANDBOXER:-"podsandbox"}

containerd_runtime_type="io.containerd.kata-${KATA_HYPERVISOR}.v2"

containerd_shim_path="$(command -v containerd-shim || true)"

#containerd config file
tmp_dir=$(mktemp -t -d test-cri-containerd.XXXX)
readonly tmp_dir
export REPORT_DIR="${tmp_dir}"
readonly CONTAINERD_CONFIG_FILE="${tmp_dir}/test-containerd-config.toml"
readonly CONTAINERD_CONF_D="${tmp_dir}/conf.d"
readonly CONTAINERD_SYSTEM_FRAGMENT_PREFIX="${CONTAINERD_SYSTEM_FRAGMENT_PREFIX:-ZZ-cri-integration-test-}"
readonly CONTAINERD_CONFIG_FILE_TEMP="${CONTAINERD_CONFIG_FILE}.temp"
readonly default_containerd_config="/etc/containerd/config.toml"
readonly default_containerd_config_backup="${CONTAINERD_CONFIG_FILE}.backup"
readonly kata_config="/etc/kata-containers/configuration.toml"
readonly kata_config_backup="${kata_config}.backup"

function ci_config() {
	sudo mkdir -p "$(dirname "${kata_config}")"
	[[ -f "${kata_config}" ]] && sudo cp "${kata_config}" "${kata_config_backup}"
	# shellcheck disable=SC2154
	sudo cp -f "${KATA_CONFIG_PATH}" "${kata_config}"

	source /etc/os-release || source /usr/lib/os-release
	ID=${ID:-""}
	if [[ "${ID}" == ubuntu ]]; then
		# https://github.com/kata-containers/tests/issues/352
		if [[ -n "${FACTORY_TEST}" ]]; then
			# Handle both commented and uncommented enable_template
			sudo sed -i -e 's/^#\?enable_template.*$/enable_template = true/g' "${kata_config}"
			echo "init vm template"
			sudo -E PATH="${PATH}" "${RUNTIME}" factory init
		fi
	fi

	echo "enable debug for kata-runtime"
	# Handle both commented and uncommented enable_debug
	sudo sed -i 's/^#\?enable_debug = .*$/enable_debug = true/g' "${kata_config}"
}

function ci_cleanup() {
	source /etc/os-release || source /usr/lib/os-release

	if [[ -n "${FACTORY_TEST}" ]]; then
		echo "destroy vm template"
		sudo -E PATH="${PATH}" "${RUNTIME}" factory destroy
	fi

	if [[ -e "${default_containerd_config_backup}" ]]; then
		echo "restore containerd config"
		sudo systemctl stop containerd
		sudo rm -f /etc/containerd/conf.d/"${CONTAINERD_SYSTEM_FRAGMENT_PREFIX}"*.toml
		sudo cp "${default_containerd_config_backup}" "${default_containerd_config}"
	fi

	if [[ -f "${kata_config_backup}" ]]; then
		sudo mv "${kata_config_backup}" "${kata_config}"
	else
		sudo rm "${kata_config}"
	fi
}

function generate_runtime_options_block() {
	local runtime="${1:?}"
	local pluginid="${2:?}"
	local runtime_config_path="${3:-}"
	local runtime_binary_path="${4:-}"

	if [[ "${runtime}" == "runc" ]]; then
		return 0
	fi

	cat << EOF
        [plugins.${pluginid}.containerd.runtimes.${runtime}.options]
          ConfigPath = "${runtime_config_path}"
          BinaryName = "${runtime_binary_path}"
EOF
}

function create_containerd_config() {
	local runtime="$1"
	# kata_annotations is set to 1 if caller want containerd setup with
	# kata annotations support.
	local kata_annotations=${2-0}
	[[ -n "${runtime}" ]] || die "need runtime to create config"

	local runtime_type="${containerd_runtime_type}"

	local runtime_config_path="${kata_config}"

	local containerd_runtime
	containerd_runtime=$(command -v "containerd-shim-${runtime}-v2")
	local runtime_binary_path="${containerd_runtime}"

	if [[ "${runtime}" == "runc" ]]; then
		runtime_type="io.containerd.runc.v2"
		runtime_config_path=""
		runtime_binary_path=""
	fi

	local cfg_schema
	cfg_schema="$(_containerd_blob_schema_version "$(PATH="${PATH}:/usr/local/bin:/usr/local/sbin" containerd config default 2>/dev/null || true)")"
	[[ "${cfg_schema}" =~ ^[0-9]+$ ]] || die "could not read containerd config schema from config default"

	info "Kata Config Path ${runtime_config_path}, Runtime Binary Name ${runtime_binary_path}"

	if [[ "${cfg_schema}" -ge 3 ]]; then
		# containerd v2.x (schema v3+): base config (imports conf.d) plus drop-in
		# fragments for sockets, debug, runtime and the linux shim.
		local pluginid=\"io.containerd.cri.v1.runtime\"
		local runtime_options_block=""
		runtime_options_block="$(generate_runtime_options_block \
			"${runtime}" "${pluginid}" "${runtime_config_path}" "${runtime_binary_path}")"

		mkdir -p "${CONTAINERD_CONF_D}"
		containerd_render_config_default_with_imports "${CONTAINERD_CONFIG_FILE}" "${CONTAINERD_CONF_D}"
		containerd_emit_rootful_api_socket_overrides "${cfg_schema}" > "${CONTAINERD_CONF_D}/10-cri-test-api-sockets.toml"

		cat <<'EOF' > "${CONTAINERD_CONF_D}/20-cri-test-debug.toml"
[debug]
  level = "debug"
EOF

		cat << EOF > "${CONTAINERD_CONF_D}/30-cri-test-runtime.toml"
[plugins]
  [plugins.${pluginid}]
    [plugins.${pluginid}.containerd]
        default_runtime_name = "${runtime}"
      [plugins.${pluginid}.containerd.runtimes.${runtime}]
        runtime_type = "${runtime_type}"
        sandboxer = "${SANDBOXER}"
        $( [[ "${kata_annotations}" -eq 1 ]] && \
        echo 'pod_annotations = ["io.katacontainers.*"]' && \
        echo '        container_annotations = ["io.katacontainers.*"]'
        )
${runtime_options_block}
EOF

		if [[ -n "${containerd_shim_path}" ]]; then
			cat << EOF > "${CONTAINERD_CONF_D}/40-cri-test-linux.toml"
[plugins.linux]
  shim = "${containerd_shim_path}"
EOF
		else
			rm -f "${CONTAINERD_CONF_D}/40-cri-test-linux.toml"
		fi
	else
		# containerd v1.x (schema v2): conf.d is not honoured the same way, so
		# write a single complete, self-contained config file.  The v1.x default
		# API sockets are already root-owned, so no socket override is needed.
		local pluginid=\"io.containerd.grpc.v1.cri\"
		local runtime_options_block=""
		runtime_options_block="$(generate_runtime_options_block \
			"${runtime}" "${pluginid}" "${runtime_config_path}" "${runtime_binary_path}")"

		mkdir -p "${CONTAINERD_CONF_D}"
		cat << EOF > "${CONTAINERD_CONFIG_FILE}"
version = 2

[debug]
  level = "debug"

[plugins]
  [plugins.${pluginid}]
    [plugins.${pluginid}.containerd]
        default_runtime_name = "${runtime}"
      [plugins.${pluginid}.containerd.runtimes.${runtime}]
        runtime_type = "${runtime_type}"
        $( [[ "${kata_annotations}" -eq 1 ]] && \
        echo 'pod_annotations = ["io.katacontainers.*"]' && \
        echo '        container_annotations = ["io.katacontainers.*"]'
        )
${runtime_options_block}
EOF
	fi
}

# Push the test containerd config to the host paths used by crictl.
function deploy_cri_integration_containerd_host_config() {
	sudo cp "${default_containerd_config}" "${default_containerd_config_backup}"

	local cfg_schema
	cfg_schema="$(_containerd_blob_schema_version "$(PATH="${PATH}:/usr/local/bin:/usr/local/sbin" containerd config default 2>/dev/null || true)")"

	if [[ "${cfg_schema}" =~ ^[0-9]+$ ]] && [[ "${cfg_schema}" -ge 3 ]]; then
		# containerd v2.x (schema v3+): base config (imports conf.d) + drop-ins.
		local host_main_fragment="${REPORT_DIR}/containerd-main-for-host.toml"

		containerd_render_config_default_with_imports "${host_main_fragment}" "/etc/containerd/conf.d"
		sudo mkdir -p /etc/containerd/conf.d
		sudo cp "${host_main_fragment}" "${default_containerd_config}"

		shopt -s nullglob
		for f in "${CONTAINERD_CONF_D}"/*.toml; do
			sudo cp "${f}" "/etc/containerd/conf.d/${CONTAINERD_SYSTEM_FRAGMENT_PREFIX}$(basename "${f}")"
		done
		shopt -u nullglob
	else
		# containerd v1.x (schema v2): deploy the complete, self-contained config.
		sudo cp "${CONTAINERD_CONFIG_FILE}" "${default_containerd_config}"
	fi

	restart_containerd_service
}

function restore_cri_integration_containerd_host_config() {
	sudo rm -f /etc/containerd/conf.d/"${CONTAINERD_SYSTEM_FRAGMENT_PREFIX}"*.toml
	sudo cp "${default_containerd_config_backup}" "${default_containerd_config}"
	restart_containerd_service
}

function cleanup() {
	ci_cleanup
	[[ -d "${tmp_dir}" ]] && rm -rf "${tmp_dir}"
}

trap cleanup EXIT

function err_report() {
	local log_file="${REPORT_DIR}/containerd.log"
	if [[ -f "${log_file}" ]]; then
		echo "::group::ERROR: containerd log :"
		echo "-------------------------------------"
		cat "${log_file}"
		echo "-------------------------------------"
		echo "::endgroup::"
	fi
	echo "::group::ERROR: Kata Containers logs : "
	echo "-------------------------------------"
	sudo journalctl -xe -t kata
	echo "-------------------------------------"
	echo "::endgroup::"
}


function check_daemon_setup() {
	info "containerd(cri): Check daemon works with runc"
	# Use podsandbox for the runc sanity check: the shim sandboxer has a known
	# containerd-side bug where the OCI spec is not populated before NewBundle is
	# called, so config.json is never written and containerd-shim-runc-v2 fails.
	# See https://github.com/containerd/containerd/issues/11640
	# This check only verifies that containerd + runc are functional before the
	# real kata tests run, so the sandboxer choice doesn't matter here.
	local SANDBOXER="podsandbox"
	create_containerd_config "runc"

	# containerd cri-integration will modify the passed in config file. Let's
	# give it a temp one.
	cp "${CONTAINERD_CONFIG_FILE}" "${CONTAINERD_CONFIG_FILE_TEMP}"
	# in some distros(AlibabaCloud), there is no btrfs-devel package available,
	# so pass GO_BUILDTAGS="no_btrfs" to make to not use btrfs.
	if ! sudo -E PATH="${PATH}:/usr/local/bin" \
		REPORT_DIR="${REPORT_DIR}" \
		FOCUS="TestImageLoad" \
		RUNTIME="" \
		CONTAINERD_CONFIG_FILE="${CONTAINERD_CONFIG_FILE_TEMP}" \
		make GO_BUILDTAGS="no_btrfs" -e cri-integration
	then
		echo "::group::ERROR: cri-integration sanity-check config"
		echo "-------------------------------------"
		cat "${CONTAINERD_CONFIG_FILE_TEMP}" || true
		echo "-------------------------------------"
		echo "::endgroup::"
		err_report
		return 1
	fi
}

function testContainerStart() {
	# no_container_yaml set to 1 will not create container_yaml
	# because caller has created its own container_yaml.
	no_container_yaml=${1:-0}

	local pod_yaml=${REPORT_DIR}/pod.yaml
	local container_yaml=${REPORT_DIR}/container.yaml
	local image="busybox:latest"

	cat << EOF > "${pod_yaml}"
metadata:
  name: busybox-sandbox1
  namespace: default
  uid: busybox-sandbox1-uid
EOF

	#TestContainerSwap has created its own container_yaml.
	if [[ "${no_container_yaml}" -ne 1 ]]; then
		cat << EOF > "${container_yaml}"
metadata:
  name: busybox-killed-vmm
  namespace: default
  uid: busybox-killed-vmm-uid
image:
  image: "${image}"
command:
- top
EOF
	fi

	deploy_cri_integration_containerd_host_config

	sudo crictl pull "${image}"
	podid=$(sudo crictl --timeout=5s runp "${pod_yaml}")
	cid=$(sudo crictl create "${podid}" "${container_yaml}" "${pod_yaml}")
	sudo crictl start "${cid}"
}

function testContainerStop() {
	info "show pod ${podid}"
	sudo crictl --timeout=20s pods --id "${podid}"
	info "stop pod ${podid}"
	sudo crictl --timeout=20s stopp "${podid}"
	info "remove pod ${podid}"
	sudo crictl --timeout=20s rmp "${podid}"

	restore_cri_integration_containerd_host_config
}

function TestKilledVmmCleanup() {
	if [[ "${KATA_HYPERVISOR}" != "qemu" ]]; then
		info "TestKilledVmmCleanup is skipped for ${KATA_HYPERVISOR}, only QEMU is currently tested"
		return 0
	fi

	info "test killed vmm cleanup"

	testContainerStart

	qemu_pid=$(pgrep -f qemu || true)
	info "kill qemu ${qemu_pid}"
	sudo kill -SIGKILL "${qemu_pid}"
	# sleep to let shimv2 exit
	sleep 1
	remained=$(pgrep -f shimv2 || true)
	[[ -z "${remained}" ]] || die "found remaining shimv2 process ${remained}"

	testContainerStop

	info "stop containerd"
}

function TestContainerMemoryUpdate() {
	# there's no need to set true for enable_virtio_mem  in dragonball
	# As it can support virtio-mem by default.
	if [[ "${KATA_HYPERVISOR}" == "dragonball" ]]; then
		# Currently, dragonball fails at decrease memory, just test increasing memory.
		# We'll re-enable it as soon as we get it to work.
		# Reference: https://github.com/kata-containers/kata-containers/issues/8804
		# DoContainerMemoryUpdate 0
		info "TestContainerMemoryUpdate skipped for dragonball"
	fi

	if [[ "${KATA_HYPERVISOR}" == "qemu-runtime-rs" ]]; then
		# Remove TestContainerMemoryUpdate from passing_test
		info "TestContainerMemoryUpdate skipped for qemu with runtime-rs"
		info "Please check out https://github.com/kata-containers/kata-containers/issues/9375"
		return
	elif [[ "${KATA_HYPERVISOR}" != "qemu" ]] || [[ "${ARCH}" == "ppc64le" ]]; then
		return
	fi

	for virtio_mem_enabled in 1 0; do
		# On s390x, only run the test when virtio_mem is enabled
		if [[ "${ARCH}" == "s390x" ]] && [[ ${virtio_mem_enabled} -eq 0 ]]; then
			continue
		fi
		PrepareContainerMemoryUpdate "${virtio_mem_enabled}"
		DoContainerMemoryUpdate "${virtio_mem_enabled}"
	done
}

function PrepareContainerMemoryUpdate() {
	test_virtio_mem=$1

	if [[ "${test_virtio_mem}" -eq 1 ]]; then
		if [[ "${ARCH}" != "x86_64" ]] && [[ "${ARCH}" != "aarch64" ]] && [[ "${ARCH}" != "s390x" ]]; then
			return
		fi
		info "Test container memory update with virtio-mem"

		# Handle both commented and uncommented enable_virtio_mem
		sudo sed -i -e 's/^#\?enable_virtio_mem.*$/enable_virtio_mem = true/g' "${kata_config}"
	else
		info "Test container memory update without virtio-mem"

		# Set to false instead of commenting out
		sudo sed -i -e 's/^#\?enable_virtio_mem.*$/enable_virtio_mem = false/g' "${kata_config}"
	fi
}

function DoContainerMemoryUpdate() {
	descrease_memory=$1

	# start a test container
	testContainerStart

	vm_size=$(($(sudo crictl exec "${cid}" cat /proc/meminfo | grep "MemTotal:" | awk '{print $2}')*1024))
	if [[ "${vm_size}" -gt $((2*1024*1024*1024)) ]] || [[ "${vm_size}" -lt $((2*1024*1024*1024-128*1024*1024)) ]]; then
		testContainerStop
		die "The VM memory size ${vm_size} before update is not right"
	fi

	sudo crictl update --memory $((2*1024*1024*1024)) "${cid}"
	sleep 1

	vm_size=$(($(sudo crictl exec "${cid}" cat /proc/meminfo | grep "MemTotal:" | awk '{print $2}')*1024))
	if [[ "${vm_size}" -gt $((4*1024*1024*1024)) ]] || [[ "${vm_size}" -lt $((4*1024*1024*1024-128*1024*1024)) ]]; then
		testContainerStop
		die "The VM memory size ${vm_size} after increase is not right"
	fi

	if [[ "${descrease_memory}" -eq 1 ]]; then
		sudo crictl update --memory $((1*1024*1024*1024)) "${cid}"
		sleep 1

		vm_size=$(($(sudo crictl exec "${cid}" cat /proc/meminfo | grep "MemTotal:" | awk '{print $2}')*1024))
		if [[ "${vm_size}" -gt $((3*1024*1024*1024)) ]] || [[ "${vm_size}" -lt $((3*1024*1024*1024-128*1024*1024)) ]]; then
			testContainerStop
			die "The VM memory size ${vm_size} after decrease is not right"
		fi
	fi

	# stop the test container
	testContainerStop
}

function getContainerSwapInfo() {
	swap_size=$(($(sudo crictl exec "${cid}" cat /proc/meminfo | grep "SwapTotal:" | awk '{print $2}')*1024))
	# NOTE: these below two checks only works on cgroup v1
	swappiness=$(sudo crictl exec "${cid}" cat /sys/fs/cgroup/memory/memory.swappiness)
	swap_in_bytes=$(sudo crictl exec "${cid}" cat /sys/fs/cgroup/memory/memory.memsw.limit_in_bytes)
}

function TestContainerSwap() {
	if [[ "${KATA_HYPERVISOR}" != "qemu" ]] || [[ "${ARCH}" != "x86_64" ]]; then
		return
	fi

	local container_yaml=${REPORT_DIR}/container.yaml
	local image="busybox:latest"

	info "Test container with guest swap"

	create_containerd_config "kata-${KATA_HYPERVISOR}" 1
	# Handle both commented and uncommented enable_guest_swap
	sudo sed -i -e 's/^#\?enable_guest_swap.*$/enable_guest_swap = true/g' "${kata_config}"

	# Test without swap device
	testContainerStart
	getContainerSwapInfo
	# Current default swappiness is 60
	if [[ "${swappiness}" -ne 60 ]]; then
		testContainerStop
		die "The VM swappiness ${swappiness} without swap device is not right"
	fi
	if [[ "${swap_in_bytes}" -lt 1125899906842624 ]]; then
		testContainerStop
		die "The VM swap_in_bytes ${swap_in_bytes} without swap device is not right"
	fi
	if [[ "${swap_size}" -ne 0 ]]; then
		testContainerStop
		die "The VM swap size ${swap_size} without swap device is not right"
	fi
	testContainerStop

	# Test with swap device
	cat << EOF > "${container_yaml}"
metadata:
  name: busybox-swap
  namespace: default
  uid: busybox-swap-uid
annotations:
  io.katacontainers.container.resource.swappiness: "100"
  io.katacontainers.container.resource.swap_in_bytes: "1610612736"
linux:
  resources:
    memory_limit_in_bytes: 1073741824
image:
  image: "${image}"
command:
- top
EOF

	testContainerStart 1
	getContainerSwapInfo
	testContainerStop

	if [[ "${swappiness}" -ne 100 ]]; then
		die "The VM swappiness ${swappiness} with swap device is not right"
	fi
	if [[ "${swap_in_bytes}" -ne 1610612736 ]]; then
		die "The VM swap_in_bytes ${swap_in_bytes} with swap device is not right"
	fi
	if [[ "${swap_size}" -ne 536870912 ]]; then
		die "The VM swap size ${swap_size} with swap device is not right"
	fi

	# Test without swap_in_bytes
	cat << EOF > "${container_yaml}"
metadata:
  name: busybox-swap
  namespace: default
  uid: busybox-swap-uid
annotations:
  io.katacontainers.container.resource.swappiness: "100"
linux:
  resources:
    memory_limit_in_bytes: 1073741824
image:
  image: "${image}"
command:
- top
EOF

	testContainerStart 1
	getContainerSwapInfo
	testContainerStop

	if [[ "${swappiness}" -ne 100 ]]; then
		die "The VM swappiness ${swappiness} without swap_in_bytes is not right"
	fi
	# swap_in_bytes is not set, it should be a value that bigger than 1125899906842624
	if [[ "${swap_in_bytes}" -lt 1125899906842624 ]]; then
		die "The VM swap_in_bytes ${swap_in_bytes} without swap_in_bytes is not right"
	fi
	if [[ "${swap_size}" -ne 1073741824 ]]; then
		die "The VM swap size ${swap_size} without swap_in_bytes is not right"
	fi

	# Test without memory_limit_in_bytes
	cat << EOF > "${container_yaml}"
metadata:
  name: busybox-swap
  namespace: default
  uid: busybox-swap-uid
annotations:
  io.katacontainers.container.resource.swappiness: "100"
image:
  image: "${image}"
command:
- top
EOF

	testContainerStart 1
	getContainerSwapInfo
	testContainerStop

	if [[ "${swappiness}" -ne 100 ]]; then
		die "The VM swappiness ${swappiness} without memory_limit_in_bytes is not right"
	fi
	# swap_in_bytes is not set, it should be a value that bigger than 1125899906842624
	if [[ "${swap_in_bytes}" -lt 1125899906842624 ]]; then
		die "The VM swap_in_bytes ${swap_in_bytes} without memory_limit_in_bytes is not right"
	fi
	if [[ "${swap_size}" -ne 2147483648 ]]; then
		die "The VM swap size ${swap_size} without memory_limit_in_bytes is not right"
	fi

	create_containerd_config "kata-${KATA_HYPERVISOR}"
}

# k8s may restart docker which will impact on containerd stop
function stop_containerd() {
	local tmp
	tmp=$(pgrep kubelet || true)
	[[ -n "${tmp}" ]] && sudo kubeadm reset -f

	sudo systemctl stop containerd
}

function mountLoopDevice() {
	local loop_file="$1"
	if [[ -e "${loop_file}" ]]; then
		sudo rm -f "${loop_file}"
		info "${loop_file} was removed"
	fi

	sudo dd if=/dev/zero of="${loop_file}" bs=100M count=2
	sudo mkfs.ext4 "${loop_file}"
	sudo losetup -fP "${loop_file}"
	local loinfo
	loinfo=$(sudo losetup -a | grep "${loop_file}" || true)
	local device
	device=$(echo "${loinfo}" | awk -F'[: ]' '{print $1}')
	echo "${device}"
}

function startDeviceCgroupContainers() {
	local pod_yaml=${REPORT_DIR}/device-cgroup-pod.yaml
	local container1_yaml=${REPORT_DIR}/device-cgroup-container1.yaml
	local container2_yaml=${REPORT_DIR}/device-cgroup-container2.yaml
	local image="busybox:latest"

    cat > "${pod_yaml}" <<EOF
metadata:
  name: busybox-device-cgroup-sandbox
  namespace: default
  uid: busybox-device-cgroup-sandbox-uid
EOF

    cat > "${container1_yaml}" <<EOF
metadata:
  name: busybox-device-cgroup-container1
  namespace: default
  uid: busybox-device-cgroup-container1-uid
image:
  image: ${image}
command:
- top
linux: {}
devices:
- container_path: ${loop_dev1_container_path}
  host_path: "${loop_dev1}"
  permissions: rwm
EOF

    cat > "${container2_yaml}" <<EOF
metadata:
  name: busybox-device-cgroup-container2
  namespace: default
  uid: busybox-device-cgroup-container2-uid
image:
  image: ${image}
command:
- top
linux: {}
devices:
- container_path: ${loop_dev2_container_path}
  host_path: "${loop_dev2}"
  permissions: rwm
EOF

	deploy_cri_integration_containerd_host_config

	sudo crictl pull "${image}"
	podid=$(sudo crictl --timeout=5s runp "${pod_yaml}")
	cid1=$(sudo crictl create "${podid}" "${container1_yaml}" "${pod_yaml}")
	cid2=$(sudo crictl create "${podid}" "${container2_yaml}" "${pod_yaml}")
	sudo crictl start "${cid1}"
	sudo crictl start "${cid2}"
}

function stopDeviceCgroupContainers() {
	info "show pod ${podid}"
	sudo crictl --timeout=20s pods --id "${podid}"
	info "stop pod ${podid}"
	sudo crictl --timeout=20s stopp "${podid}"
	info "remove pod ${podid}"
	sudo crictl --timeout=20s rmp "${podid}"

	restore_cri_integration_containerd_host_config
}

function TestDeviceCgroup() {
	loop_dev1=$(mountLoopDevice "/tmp/device-cgroup-1.img" | tail -n 1)
	loop_dev2=$(mountLoopDevice "/tmp/device-cgroup-2.img" | tail -n 1)
	info "Two loop devices, ${loop_dev1} and ${loop_dev2}, are created."
	loop_dev1_container_path="/dev/test-block-1"
	loop_dev2_container_path="/dev/test-block-2"

	startDeviceCgroupContainers

	local dev1_ls
	dev1_ls=$(sudo crictl exec "${cid1}" ls -l "${loop_dev1_container_path}")
	local dev1_no
	dev1_no=$(echo "${dev1_ls}" | awk '{print $5, $6}')
	local dev1_major
	dev1_major=$(echo "${dev1_no%%,*}" | tr -d ' ')
	local dev1_minor
	dev1_minor=$(echo "${dev1_no##*,}" | tr -d ' ')

	local dev2_ls
	dev2_ls=$(sudo crictl exec "${cid2}" ls -l "${loop_dev2_container_path}")
	local dev2_no
	dev2_no=$(echo "${dev2_ls}" | awk '{print $5, $6}')
	local dev2_major
	dev2_major=$(echo "${dev2_no%%,*}" | tr -d ' ')
	local dev2_minor
	dev2_minor=$(echo "${dev2_no##*,}" | tr -d ' ')

	info "\"${dev1_major}:${dev1_minor}\" is for container1, and \"${dev2_major}:${dev2_minor}\" is for container2."

	local cid1_device_cgroup
	cid1_device_cgroup=$(sudo crictl exec "${cid1}" cat /sys/fs/cgroup/devices/devices.list)
	local cid2_device_cgroup
	cid2_device_cgroup=$(sudo crictl exec "${cid2}" cat /sys/fs/cgroup/devices/devices.list)

 	if [[ ${cid1_device_cgroup} != *"b ${dev1_major}:${dev1_minor} rwm"* ]]; then
 		die "The device cgroup of container1 is expected to have loop dev1"
	fi
	info "Container1 has \"b ${dev1_major}:${dev1_minor} rwm\"."

	if [[ ${cid1_device_cgroup} == *"b ${dev2_major}:${dev2_minor} rwm"* ]]; then
  		die "The device cgroup of container1 isn't expected to have loop dev2"
 	fi
	info "Container1 doesn't have \"b ${dev2_major}:${dev2_minor} rwm\"."

  	if [[ ${cid2_device_cgroup} == *"b ${dev1_major}:${dev1_minor} rwm"* ]]; then
   		die "The device cgroup of container2 isn't expected to have loop dev2"
 	fi
	info "Container2 doesn't have \"b ${dev1_major}:${dev1_minor} rwm\"."

 	if [[ ${cid2_device_cgroup} != *"b ${dev2_major}:${dev2_minor} rwm"* ]]; then
 		die "The device cgroup of container2 is expected to have loop dev2"
 	fi
	info "Container2 has \"b ${dev2_major}:${dev2_minor} rwm\"."

 	stopDeviceCgroupContainers

 	# Umount loop devices
 	sudo losetup -d "${loop_dev1}"
 	sudo losetup -d "${loop_dev2}"
	info "Two loop devices, ${loop_dev1} and ${loop_dev2}, are umounted."
}

function main() {

	info "Clean up containers and pods"
	restart_containerd_service
	mapfile -t containers < <(sudo crictl ps --all -o json | jq -r '.containers[].id')
	for c in "${containers[@]}"; do
		sudo crictl rm -f "${c}"
	done
	mapfile -t pods < <(sudo crictl pods -o json | jq -r '.items[].id')
	for p in "${pods[@]}"; do
		sudo crictl rmp -f "${p}"
	done

	info "Stop crio service"
	systemctl is-active --quiet crio && sudo systemctl stop crio

	info "Stop containerd service"
	systemctl is-active --quiet containerd && stop_containerd

	# Configure enviroment if running in CI
	ci_config

	pushd "containerd"

	export_go_toolchain_for_containerd_source_builds

	# Make sure the right artifacts are going to be built
	sudo -E make clean

	# the latest containerd had an issue for its e2e test, thus we should do the following
	# fix to workaround this issue. For much info about this issue, please see:
	# https://github.com/containerd/containerd/pull/11240
	# Once this pr was merged and release new version, we can remove this workaround.
	# shellcheck disable=SC2016
	sed -i 's/cat "\${config_file}"/cat "\${CONTAINERD_CONFIG_FILE}"/' script/test/utils.sh

	check_daemon_setup

	info "containerd(cri): testing using runtime: ${containerd_runtime_type}"

	create_containerd_config "kata-${KATA_HYPERVISOR}"

	info "containerd(cri): Running cri-integration"

	# TestContainerRestart is excluded: creating a new container in the same
	# sandbox VM after the previous container has exited and been removed has
	# never been supported by kata-containers (neither with the go-based nor
	# the rust-based runtime).  The kata VM shuts down when its last container
	# is removed, so any attempt to start a new container in the same sandbox
	# fails.  This test exercises a use-case kata does not currently support.
	passing_test="TestContainerStats|TestContainerListStatsWithIdFilter|TestContainerListStatsWithIdSandboxIdFilter|TestDuplicateName|TestImageLoad|TestImageFSInfo|TestSandboxCleanRemove"

	if [[ "${KATA_HYPERVISOR}" == "clh-runtime-rs" || \
		"${KATA_HYPERVISOR}" == "qemu" ]]; then
		issue="https://github.com/kata-containers/tests/issues/2318"
		info "${KATA_HYPERVISOR} fails with TestContainerListStatsWithSandboxIdFilter }"
		info "see ${issue}"
	else
		passing_test="${passing_test}|TestContainerListStatsWithSandboxIdFilter"
	fi

	# in some distros(AlibabaCloud), there is no btrfs-devel package available,
	# so pass GO_BUILDTAGS="no_btrfs" to make to not use btrfs.
	# containerd cri-integration will modify the passed in config file. Let's
	# give it a temp one.
	cp "${CONTAINERD_CONFIG_FILE}" "${CONTAINERD_CONFIG_FILE_TEMP}"
	sudo -E PATH="${PATH}:/usr/local/bin" \
		REPORT_DIR="${REPORT_DIR}" \
		FOCUS="^(${passing_test})$" \
		RUNTIME="" \
		CONTAINERD_CONFIG_FILE="${CONTAINERD_CONFIG_FILE_TEMP}" \
		make GO_BUILDTAGS="no_btrfs" -e cri-integration

	# trap error for print containerd log,
	# containerd's `cri-integration` will print the log itself.
	trap err_report ERR

	# TestContainerSwap is currently failing with GHA.
	# Let's re-enable it as soon as we get it to work.
	# Reference: https://github.com/kata-containers/kata-containers/issues/7410
	# TestContainerSwap

	TestContainerMemoryUpdate

	if [[ "${ARCH}" != "ppc64le" ]]; then
		if [[ "${KATA_HYPERVISOR}" == "qemu-runtime-rs" ]]; then
			info "TestKilledVmmCleanup and TestDeviceCgroup skipped for qemu with runtime-rs"
			info "Please check out https://github.com/kata-containers/kata-containers/issues/9375"
			return
		else
			TestKilledVmmCleanup

			info "Skipping TestDeviceCgroup till the test is adapted to cgroupsv2"
			#TestDeviceCgroup
		fi
	fi

	popd
}

main