diff --git a/tests/common.bash b/tests/common.bash index 75598b24ec..d71b3f43ce 100644 --- a/tests/common.bash +++ b/tests/common.bash @@ -7,6 +7,9 @@ # This file contains common functions that # are being used by our metrics and integration tests +this_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +export repo_root_dir="$(cd "${this_script_dir}/../" && pwd)" + # Kata tests directory used for storing various test-related artifacts. KATA_TESTS_BASEDIR="${KATA_TESTS_BASEDIR:-/var/log/kata-tests}" @@ -266,19 +269,6 @@ function restart_containerd_service() { return 0 } -function create_symbolic_links() { - local KATA_HYPERVISOR="${1}" - - local link_configuration_file="/opt/kata/share/defaults/kata-containers/configuration.toml" - local source_configuration_file="/opt/kata/share/defaults/kata-containers/configuration-${KATA_HYPERVISOR}.toml" - - if [ "${KATA_HYPERVISOR}" != 'qemu' ] && [ "${KATA_HYPERVISOR}" != 'clh' ]; then - die "Failed to set the configuration.toml: '${KATA_HYPERVISOR}' is not recognized as a valid hypervisor name." - fi - - sudo ln -sf "${source_configuration_file}" "${link_configuration_file}" -} - # Configures containerd function overwrite_containerd_config() { containerd_config="/etc/containerd/config.toml" @@ -312,13 +302,20 @@ function install_kata() { popd # create symbolic links to kata components - for b in "${katadir}/bin/*" ; do + for b in "${katadir}"/bin/* ; do sudo ln -sf "${b}" "${local_bin_dir}/$(basename $b)" done + if [[ ${KATA_HYPERVISOR} == "dragonball" ]]; then + sudo ln -sf "${katadir}/runtime-rs/bin/containerd-shim-kata-v2" "${local_bin_dir}/containerd-shim-kata-${KATA_HYPERVISOR}-v2" + else + sudo ln -sf "${katadir}/bin/containerd-shim-kata-v2" "${local_bin_dir}/containerd-shim-kata-${KATA_HYPERVISOR}-v2" + fi + + sudo ln -sf ${katadir}/share/defaults/kata-containers/configuration-${KATA_HYPERVISOR}.toml ${katadir}/share/defaults/kata-containers/configuration.toml + check_containerd_config_for_kata restart_containerd_service - install_checkmetrics } function check_containerd_config_for_kata() { @@ -336,3 +333,83 @@ function check_containerd_config_for_kata() { overwrite_containerd_config fi } + +function ensure_yq() { + : "${GOPATH:=${GITHUB_WORKSPACE:-$HOME/go}}" + export GOPATH + export PATH="${GOPATH}/bin:${PATH}" + INSTALL_IN_GOPATH=true "${repo_root_dir}/ci/install_yq.sh" +} + +# dependency: What we want to get the version from the versions.yaml file +function get_from_kata_deps() { + local dependency="$1" + versions_file="${repo_root_dir}/versions.yaml" + + command -v yq &>/dev/null || die 'yq command is not in your $PATH' + result=$("yq" read -X "$versions_file" "$dependency") + [ "$result" = "null" ] && result="" + echo "$result" +} + +# project: org/repo format +# base_version: ${major}.${minor} +function get_latest_patch_release_from_a_github_project() { + project="${1}" + base_version="${2}" + + curl --silent https://api.github.com/repos/${project}/releases | jq -r .[].tag_name | grep "^${base_version}.[0-9]*$" -m1 +} + +# base_version: The version to be intalled in the ${major}.${minor} format +function clone_cri_containerd() { + base_version="${1}" + + project="containerd/containerd" + version=$(get_latest_patch_release_from_a_github_project "${project}" "${base_version}") + + rm -rf containerd + git clone -b ${version} https://github.com/${project} +} + +# project: org/repo format +# version: the version of the tarball that will be downloaded +# tarball-name: the name of the tarball that will be downloaded +function download_github_project_tarball() { + project="${1}" + version="${2}" + tarball_name="${3}" + + wget https://github.com/${project}/releases/download/${version}/${tarball_name} +} + +# base_version: The version to be intalled in the ${major}.${minor} format +function install_cri_containerd() { + base_version="${1}" + + project="containerd/containerd" + version=$(get_latest_patch_release_from_a_github_project "${project}" "${base_version}") + + tarball_name="cri-containerd-cni-${version//v}-linux-$(${repo_root_dir}/tests/kata-arch.sh -g).tar.gz" + + download_github_project_tarball "${project}" "${version}" "${tarball_name}" + sudo tar -xvf "${tarball_name}" -C / + rm -f "${tarball_name}" + + sudo mkdir -p /etc/containerd + containerd config default | sudo tee /etc/containerd/config.toml +} + +# base_version: The version to be intalled in the ${major}.${minor} format +function install_cri_tools() { + base_version="${1}" + + project="kubernetes-sigs/cri-tools" + version=$(get_latest_patch_release_from_a_github_project "${project}" "${base_version}") + + tarball_name="crictl-${version}-linux-$(${repo_root_dir}/tests/kata-arch.sh -g).tar.gz" + + download_github_project_tarball "${project}" "${version}" "${tarball_name}" + sudo tar -xvf "${tarball_name}" -C /usr/local/bin + rm -f "${tarball_name}" +} diff --git a/tests/install_go.sh b/tests/install_go.sh new file mode 100755 index 0000000000..3827bb7b3a --- /dev/null +++ b/tests/install_go.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# +# Copyright (c) 2018-2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail + +tmp_dir=$(mktemp -d -t install-go-tmp.XXXXXXXXXX) +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +script_name="$(basename "${BASH_SOURCE[0]}")" +force="" +USE_VERSIONS_FILE="" +PROJECT="Kata Containers" + +source "${script_dir}/common.bash" + +install_dest="/usr/local/" + +function finish() { + rm -rf "$tmp_dir" +} + +function usage(){ + exit_code="$1" + cat < + +Args: + : Install a specific go version. + +Example: +${script_name} 1.10 + +Options +-d : destination path, path where go will be installed. +-f : Force remove old go version and install the specified one. +-h : Show this help +-p : Install go defined in ${PROJECT} versions file. + +EOF + + exit "$exit_code" +} + +trap finish EXIT + +pushd "${tmp_dir}" + +while getopts "d:fhp" opt +do + case $opt in + d) install_dest="${OPTARG}" ;; + f) force="true" ;; + h) usage 0 ;; + p) USE_VERSIONS_FILE="true" ;; + esac +done + +shift $(( $OPTIND - 1 )) + +go_version="${1:-""}" + +if [ -z "$go_version" ] && [ "${USE_VERSIONS_FILE}" = "true" ] ;then + go_version=$(get_from_kata_deps "languages.golang.meta.newest-version") +fi + +if [ -z "$go_version" ];then + echo "Missing go version or -p option" + usage 0 +fi + +if command -v go; then + [[ "$(go version)" == *"go${go_version}"* ]] && \ + info "Go ${go_version} already installed" && \ + exit + if [ "${force}" = "true" ]; then + info "removing $(go version)" + sudo rm -rf "${install_dest}/go" + else + die "$(go version) is installed, use -f or remove it before install go ${go_version}" + fi +fi + +goarch=$("${repo_root_dir}/tests/kata-arch.sh" --golang) + +info "Download go version ${go_version}" +kernel_name=$(uname -s) +curl -OL "https://storage.googleapis.com/golang/go${go_version}.${kernel_name,,}-${goarch}.tar.gz" +info "Install go" +mkdir -p "${install_dest}" +sudo tar -C "${install_dest}" -xzf "go${go_version}.${kernel_name,,}-${goarch}.tar.gz" +popd diff --git a/tests/integration/cri-containerd/gha-run.sh b/tests/integration/cri-containerd/gha-run.sh index 259c2a02cc..c36c966057 100755 --- a/tests/integration/cri-containerd/gha-run.sh +++ b/tests/integration/cri-containerd/gha-run.sh @@ -14,13 +14,51 @@ cri_containerd_dir="$(dirname "$(readlink -f "$0")")" source "${cri_containerd_dir}/../../common.bash" function install_dependencies() { - return 0 + info "Installing the dependencies needed for running the cri-containerd tests" + + # Dependency list of projects that we can rely on the system packages + # - build-essential + # - Theoretically we only need `make`, but doesn't hurt to install + # the whole build-essential group + # - jq + # - podman-docker + # - one of the tests rely on docker to pull an image. + # we've decided to go for podman, instead, as it does *not* bring + # containerd as a dependency + declare -a system_deps=( + build-essential + jq + podman-docker + ) + + sudo apt-get update + sudo apt-get -y install "${system_deps[@]}" + + ensure_yq + ${repo_root_dir}/tests/install_go.sh -p + + # Dependency list of projects that we can install them + # directly from their releases on GitHub: + # - containerd + # - cri-container-cni release tarball already includes CNI plugins + # - cri-tools + declare -a github_deps + github_deps[0]="cri_containerd:$(get_from_kata_deps "externals.containerd.${CONTAINERD_VERSION}")" + github_deps[1]="cri_tools:$(get_from_kata_deps "externals.critools.latest")" + + for github_dep in "${github_deps[@]}"; do + IFS=":" read -r -a dep <<< "${github_dep}" + install_${dep[0]} "${dep[1]}" + done + + # Clone containerd as we'll need to build it in order to run the tests + # base_version: The version to be intalled in the ${major}.${minor} format + clone_cri_containerd $(get_from_kata_deps "externals.containerd.${CONTAINERD_VERSION}") } function run() { info "Running cri-containerd tests using ${KATA_HYPERVISOR} hypervisor" - create_symbolic_links ${KATA_HYPERVISOR} return 0 } diff --git a/tests/integration/cri-containerd/integration-tests.sh b/tests/integration/cri-containerd/integration-tests.sh new file mode 100755 index 0000000000..0e4df5578f --- /dev/null +++ b/tests/integration/cri-containerd/integration-tests.sh @@ -0,0 +1,493 @@ +#!/bin/bash +# +# Copyright (c) 2017-2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +[[ "${DEBUG}" != "" ]] && set -o xtrace +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +SCRIPT_PATH=$(dirname "$(readlink -f "$0")") +source "${SCRIPT_PATH}/../../common.bash" + +# runc is installed in /usr/local/sbin/ add that path +export PATH="$PATH:/usr/local/sbin" + +# golang is installed in /usr/local/go/bin/ add that path +export PATH="$PATH:/usr/local/go/bin" + +# Runtime to be used for testing +RUNTIME=${RUNTIME:-containerd-shim-kata-v2} +FACTORY_TEST=${FACTORY_TEST:-""} +KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" +USE_DEVMAPPER="${USE_DEVMAPPER:-false}" +ARCH=$(uname -m) + +containerd_runtime_type="io.containerd.kata-${KATA_HYPERVISOR}.v2" + +containerd_shim_path="$(command -v containerd-shim)" + +#containerd config file +readonly tmp_dir=$(mktemp -t -d test-cri-containerd.XXXX) +export REPORT_DIR="${tmp_dir}" +readonly CONTAINERD_CONFIG_FILE="${tmp_dir}/test-containerd-config" +readonly CONTAINERD_CONFIG_FILE_TEMP="${CONTAINERD_CONFIG_FILE}.temp" +readonly default_containerd_config="/etc/containerd/config.toml" +readonly default_containerd_config_backup="$CONTAINERD_CONFIG_FILE.backup" +readonly kata_config="/etc/kata-containers/configuration.toml" +readonly kata_config_backup="$kata_config.backup" +readonly default_kata_config="/opt/kata/share/defaults/kata-containers/configuration.toml" + +function ci_config() { + sudo mkdir -p $(dirname "${kata_config}") + [ -f "$kata_config" ] && sudo cp "$kata_config" "$kata_config_backup" || \ + sudo cp "$default_kata_config" "$kata_config" + + source /etc/os-release || source /usr/lib/os-release + ID=${ID:-""} + if [ "$ID" == ubuntu ]; then + # https://github.com/kata-containers/tests/issues/352 + if [ -n "${FACTORY_TEST}" ]; then + sudo sed -i -e 's/^#enable_template.*$/enable_template = true/g' "${kata_config}" + echo "init vm template" + sudo -E PATH=$PATH "$RUNTIME" factory init + fi + fi + + echo "enable debug for kata-runtime" + sudo sed -i 's/^#enable_debug =/enable_debug =/g' ${kata_config} +} + +function ci_cleanup() { + source /etc/os-release || source /usr/lib/os-release + + if [ -n "${FACTORY_TEST}" ]; then + echo "destroy vm template" + sudo -E PATH=$PATH "$RUNTIME" factory destroy + fi + + if [ -e "$default_containerd_config_backup" ]; then + echo "restore containerd config" + sudo systemctl stop containerd + sudo cp "$default_containerd_config_backup" "$default_containerd_config" + fi + + [ -f "$kata_config_backup" ] && sudo mv "$kata_config_backup" "$kata_config" || \ + sudo rm "$kata_config" +} + +function create_containerd_config() { + local runtime="$1" + # kata_annotations is set to 1 if caller want containerd setup with + # kata annotations support. + local kata_annotations=${2-0} + [ -n "${runtime}" ] || die "need runtime to create config" + + local runtime_type="${containerd_runtime_type}" + if [ "${runtime}" == "runc" ]; then + runtime_type="io.containerd.runc.v2" + fi + local containerd_runtime=$(command -v "containerd-shim-${runtime}-v2") + +cat << EOF | sudo tee "${CONTAINERD_CONFIG_FILE}" +[debug] + level = "debug" +[plugins] + [plugins.cri] + [plugins.cri.containerd] + default_runtime_name = "$runtime" + [plugins.cri.containerd.runtimes.${runtime}] + runtime_type = "${runtime_type}" + $( [ $kata_annotations -eq 1 ] && \ + echo 'pod_annotations = ["io.katacontainers.*"]' && \ + echo ' container_annotations = ["io.katacontainers.*"]' + ) + [plugins.cri.containerd.runtimes.${runtime}.options] + Runtime = "${containerd_runtime}" +[plugins.linux] + shim = "${containerd_shim_path}" +EOF + +if [ "$USE_DEVMAPPER" == "true" ]; then + sudo sed -i 's|^\(\[plugins\]\).*|\1\n \[plugins.devmapper\]\n pool_name = \"contd-thin-pool\"\n base_image_size = \"4096MB\"|' ${CONTAINERD_CONFIG_FILE} + echo "Devicemapper configured" + cat "${CONTAINERD_CONFIG_FILE}" +fi + +} + +function cleanup() { + ci_cleanup + [ -d "$tmp_dir" ] && rm -rf "${tmp_dir}" +} + +trap cleanup EXIT + +function err_report() { + local log_file="${REPORT_DIR}/containerd.log" + if [ -f "$log_file" ]; then + echo "ERROR: containerd log :" + echo "-------------------------------------" + cat "${log_file}" + echo "-------------------------------------" + fi +} + + +function check_daemon_setup() { + info "containerd(cri): Check daemon works with runc" + create_containerd_config "runc" + + # containerd cri-integration will modify the passed in config file. Let's + # give it a temp one. + cp $CONTAINERD_CONFIG_FILE $CONTAINERD_CONFIG_FILE_TEMP + # in some distros(AlibabaCloud), there is no btrfs-devel package available, + # so pass GO_BUILDTAGS="no_btrfs" to make to not use btrfs. + sudo -E PATH="${PATH}:/usr/local/bin" \ + REPORT_DIR="${REPORT_DIR}" \ + FOCUS="TestImageLoad" \ + RUNTIME="" \ + CONTAINERD_CONFIG_FILE="$CONTAINERD_CONFIG_FILE_TEMP" \ + make GO_BUILDTAGS="no_btrfs" -e cri-integration +} + +function testContainerStart() { + # no_container_yaml set to 1 will not create container_yaml + # because caller has created its own container_yaml. + no_container_yaml=${1:-0} + + local pod_yaml=${REPORT_DIR}/pod.yaml + local container_yaml=${REPORT_DIR}/container.yaml + local image="busybox:latest" + + cat << EOF > "${pod_yaml}" +metadata: + name: busybox-sandbox1 + namespace: default + uid: busybox-sandbox1-uid +EOF + + #TestContainerSwap has created its own container_yaml. + if [ $no_container_yaml -ne 1 ]; then + cat << EOF > "${container_yaml}" +metadata: + name: busybox-killed-vmm + namespace: default + uid: busybox-killed-vmm-uid +image: + image: "$image" +command: +- top +EOF + fi + + sudo cp "$default_containerd_config" "$default_containerd_config_backup" + sudo cp $CONTAINERD_CONFIG_FILE "$default_containerd_config" + + restart_containerd_service + + sudo crictl pull $image + podid=$(sudo crictl runp $pod_yaml) + cid=$(sudo crictl create $podid $container_yaml $pod_yaml) + sudo crictl start $cid +} + +function testContainerStop() { + info "stop pod $podid" + sudo crictl stopp $podid + info "remove pod $podid" + sudo crictl rmp $podid + + sudo cp "$default_containerd_config_backup" "$default_containerd_config" + restart_containerd_service +} + +function TestKilledVmmCleanup() { + if [[ "${KATA_HYPERVISOR}" != "qemu" ]]; then + info "TestKilledVmmCleanup is skipped for ${KATA_HYPERVISOR}, only QEMU is currently tested" + return 0 + fi + + info "test killed vmm cleanup" + + testContainerStart + + qemu_pid=$(ps aux|grep qemu|grep -v grep|awk '{print $2}') + info "kill qemu $qemu_pid" + sudo kill -SIGKILL $qemu_pid + # sleep to let shimv2 exit + sleep 1 + remained=$(ps aux|grep shimv2|grep -v grep || true) + [ -z $remained ] || die "found remaining shimv2 process $remained" + + testContainerStop + + info "stop containerd" +} + +function TestContainerMemoryUpdate() { + if [[ "${KATA_HYPERVISOR}" != "qemu" ]] || [[ "${ARCH}" == "ppc64le" ]] || [[ "${ARCH}" == "s390x" ]]; then + return + fi + + test_virtio_mem=$1 + + if [ $test_virtio_mem -eq 1 ]; then + if [[ "$ARCH" != "x86_64" ]]; then + return + fi + info "Test container memory update with virtio-mem" + + sudo sed -i -e 's/^#enable_virtio_mem.*$/enable_virtio_mem = true/g' "${kata_config}" + else + info "Test container memory update without virtio-mem" + + sudo sed -i -e 's/^enable_virtio_mem.*$/#enable_virtio_mem = true/g' "${kata_config}" + fi + + testContainerStart + + vm_size=$(($(sudo crictl exec $cid cat /proc/meminfo | grep "MemTotal:" | awk '{print $2}')*1024)) + if [ $vm_size -gt $((2*1024*1024*1024)) ] || [ $vm_size -lt $((2*1024*1024*1024-128*1024*1024)) ]; then + testContainerStop + die "The VM memory size $vm_size before update is not right" + fi + + sudo crictl update --memory $((2*1024*1024*1024)) $cid + sleep 1 + + vm_size=$(($(sudo crictl exec $cid cat /proc/meminfo | grep "MemTotal:" | awk '{print $2}')*1024)) + if [ $vm_size -gt $((4*1024*1024*1024)) ] || [ $vm_size -lt $((4*1024*1024*1024-128*1024*1024)) ]; then + testContainerStop + die "The VM memory size $vm_size after increase is not right" + fi + + if [ $test_virtio_mem -eq 1 ]; then + sudo crictl update --memory $((1*1024*1024*1024)) $cid + sleep 1 + + vm_size=$(($(sudo crictl exec $cid cat /proc/meminfo | grep "MemTotal:" | awk '{print $2}')*1024)) + if [ $vm_size -gt $((3*1024*1024*1024)) ] || [ $vm_size -lt $((3*1024*1024*1024-128*1024*1024)) ]; then + testContainerStop + die "The VM memory size $vm_size after decrease is not right" + fi + fi + + testContainerStop +} + +function getContainerSwapInfo() { + swap_size=$(($(sudo crictl exec $cid cat /proc/meminfo | grep "SwapTotal:" | awk '{print $2}')*1024)) + # NOTE: these below two checks only works on cgroup v1 + swappiness=$(sudo crictl exec $cid cat /sys/fs/cgroup/memory/memory.swappiness) + swap_in_bytes=$(sudo crictl exec $cid cat /sys/fs/cgroup/memory/memory.memsw.limit_in_bytes) +} + +function TestContainerSwap() { + if [[ "${KATA_HYPERVISOR}" != "qemu" ]] || [[ "${ARCH}" != "x86_64" ]]; then + return + fi + + local container_yaml=${REPORT_DIR}/container.yaml + local image="busybox:latest" + + info "Test container with guest swap" + + create_containerd_config "kata-${KATA_HYPERVISOR}" 1 + sudo sed -i -e 's/^#enable_guest_swap.*$/enable_guest_swap = true/g' "${kata_config}" + + # Test without swap device + testContainerStart + getContainerSwapInfo + # Current default swappiness is 60 + if [ $swappiness -ne 60 ]; then + testContainerStop + die "The VM swappiness $swappiness without swap device is not right" + fi + if [ $swap_in_bytes -lt 1125899906842624 ]; then + testContainerStop + die "The VM swap_in_bytes $swap_in_bytes without swap device is not right" + fi + if [ $swap_size -ne 0 ]; then + testContainerStop + die "The VM swap size $swap_size without swap device is not right" + fi + testContainerStop + + # Test with swap device + cat << EOF > "${container_yaml}" +metadata: + name: busybox-swap + namespace: default + uid: busybox-swap-uid +annotations: + io.katacontainers.container.resource.swappiness: "100" + io.katacontainers.container.resource.swap_in_bytes: "1610612736" +linux: + resources: + memory_limit_in_bytes: 1073741824 +image: + image: "$image" +command: +- top +EOF + + testContainerStart 1 + getContainerSwapInfo + testContainerStop + + if [ $swappiness -ne 100 ]; then + die "The VM swappiness $swappiness with swap device is not right" + fi + if [ $swap_in_bytes -ne 1610612736 ]; then + die "The VM swap_in_bytes $swap_in_bytes with swap device is not right" + fi + if [ $swap_size -ne 536870912 ]; then + die "The VM swap size $swap_size with swap device is not right" + fi + + # Test without swap_in_bytes + cat << EOF > "${container_yaml}" +metadata: + name: busybox-swap + namespace: default + uid: busybox-swap-uid +annotations: + io.katacontainers.container.resource.swappiness: "100" +linux: + resources: + memory_limit_in_bytes: 1073741824 +image: + image: "$image" +command: +- top +EOF + + testContainerStart 1 + getContainerSwapInfo + testContainerStop + + if [ $swappiness -ne 100 ]; then + die "The VM swappiness $swappiness without swap_in_bytes is not right" + fi + # swap_in_bytes is not set, it should be a value that bigger than 1125899906842624 + if [ $swap_in_bytes -lt 1125899906842624 ]; then + die "The VM swap_in_bytes $swap_in_bytes without swap_in_bytes is not right" + fi + if [ $swap_size -ne 1073741824 ]; then + die "The VM swap size $swap_size without swap_in_bytes is not right" + fi + + # Test without memory_limit_in_bytes + cat << EOF > "${container_yaml}" +metadata: + name: busybox-swap + namespace: default + uid: busybox-swap-uid +annotations: + io.katacontainers.container.resource.swappiness: "100" +image: + image: "$image" +command: +- top +EOF + + testContainerStart 1 + getContainerSwapInfo + testContainerStop + + if [ $swappiness -ne 100 ]; then + die "The VM swappiness $swappiness without memory_limit_in_bytes is not right" + fi + # swap_in_bytes is not set, it should be a value that bigger than 1125899906842624 + if [ $swap_in_bytes -lt 1125899906842624 ]; then + die "The VM swap_in_bytes $swap_in_bytes without memory_limit_in_bytes is not right" + fi + if [ $swap_size -ne 2147483648 ]; then + die "The VM swap size $swap_size without memory_limit_in_bytes is not right" + fi + + create_containerd_config "kata-${KATA_HYPERVISOR}" +} + +# k8s may restart docker which will impact on containerd stop +function stop_containerd() { + local tmp=$(pgrep kubelet || true) + [ -n "$tmp" ] && sudo kubeadm reset -f + + sudo systemctl stop containerd +} + +function main() { + + info "Stop crio service" + systemctl is-active --quiet crio && sudo systemctl stop crio + + info "Stop containerd service" + systemctl is-active --quiet containerd && stop_containerd + + # Configure enviroment if running in CI + ci_config + + pushd "containerd" + + # Make sure the right artifacts are going to be built + make clean + + check_daemon_setup + + info "containerd(cri): testing using runtime: ${containerd_runtime_type}" + + create_containerd_config "kata-${KATA_HYPERVISOR}" + + info "containerd(cri): Running cri-integration" + + + passing_test="TestContainerStats|TestContainerRestart|TestContainerListStatsWithIdFilter|TestContainerListStatsWithIdSandboxIdFilter|TestDuplicateName|TestImageLoad|TestImageFSInfo|TestSandboxCleanRemove" + + if [[ "${KATA_HYPERVISOR}" == "cloud-hypervisor" || \ + "${KATA_HYPERVISOR}" == "qemu" ]]; then + issue="https://github.com/kata-containers/tests/issues/2318" + info "${KATA_HYPERVISOR} fails with TestContainerListStatsWithSandboxIdFilter }" + info "see ${issue}" + else + passing_test="${passing_test}|TestContainerListStatsWithSandboxIdFilter" + fi + + # in some distros(AlibabaCloud), there is no btrfs-devel package available, + # so pass GO_BUILDTAGS="no_btrfs" to make to not use btrfs. + # containerd cri-integration will modify the passed in config file. Let's + # give it a temp one. + cp $CONTAINERD_CONFIG_FILE $CONTAINERD_CONFIG_FILE_TEMP + sudo -E PATH="${PATH}:/usr/local/bin" \ + REPORT_DIR="${REPORT_DIR}" \ + FOCUS="^(${passing_test})$" \ + RUNTIME="" \ + CONTAINERD_CONFIG_FILE="$CONTAINERD_CONFIG_FILE_TEMP" \ + make GO_BUILDTAGS="no_btrfs" -e cri-integration + + # trap error for print containerd log, + # containerd's `cri-integration` will print the log itself. + trap err_report ERR + + # TestContainerSwap is currently failing with GHA. + # Let's re-enable it as soon as we get it to work. + # Reference: https://github.com/kata-containers/kata-containers/issues/7410 + # TestContainerSwap + + # TODO: runtime-rs doesn't support memory update currently + if [ "$KATA_HYPERVISOR" != "dragonball" ]; then + TestContainerMemoryUpdate 1 + TestContainerMemoryUpdate 0 + fi + + TestKilledVmmCleanup + + popd +} + +main diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 85180920a8..c873a0d79c 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -9,7 +9,7 @@ set -o nounset set -o pipefail kubernetes_dir="$(dirname "$(readlink -f "$0")")" -repo_root_dir="$(cd "${kubernetes_dir}/../../../" && pwd)" +source "${kubernetes_dir}/../../common.bash" tools_dir="${repo_root_dir}/tools" function _print_cluster_name() { @@ -59,13 +59,6 @@ function get_cluster_credentials() { -n "$(_print_cluster_name)" } -function ensure_yq() { - : "${GOPATH:=${GITHUB_WORKSPACE}}" - export GOPATH - export PATH="${GOPATH}/bin:${PATH}" - INSTALL_IN_GOPATH=true "${repo_root_dir}/ci/install_yq.sh" -} - function run_tests() { platform="${1}" ensure_yq diff --git a/tests/integration/kubernetes/setup.sh b/tests/integration/kubernetes/setup.sh index 213397f83b..01e7d8c51a 100755 --- a/tests/integration/kubernetes/setup.sh +++ b/tests/integration/kubernetes/setup.sh @@ -8,7 +8,7 @@ set -o nounset set -o pipefail kubernetes_dir=$(dirname "$(readlink -f "$0")") -repo_root_dir="$(cd "${kubernetes_dir}/../../../" && pwd)" +source "${kubernetes_dir}/../../common.bash" set_runtime_class() { sed -i -e "s|runtimeClassName: kata|runtimeClassName: kata-${KATA_HYPERVISOR}|" ${kubernetes_dir}/runtimeclass_workloads/*.yaml diff --git a/tests/kata-arch.sh b/tests/kata-arch.sh new file mode 100755 index 0000000000..38146ef727 --- /dev/null +++ b/tests/kata-arch.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018-2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +typeset -r script_name=${0##*/} + +typeset -r cidir=$(dirname "$0") + +function usage() { + cat <&2; exit 1; } + + while [ $# -gt 1 ] + do + case "$1" in + -d|--default) ;; + + -g|--golang) type="golang";; + + -r|--rust) type="rust";; + + -h|--help) + usage + exit 0 + ;; + + -k|--kernel) type="kernel";; + + --) + shift + break + ;; + esac + shift + done + + local -r arch=$(uname -m) + + case "$type" in + default) echo "$arch";; + golang) arch_to_golang "$arch";; + rust) arch_to_rust "${arch}";; + kernel) arch_to_kernel "$arch";; + esac +} + +main "$@" diff --git a/tests/metrics/gha-run.sh b/tests/metrics/gha-run.sh index 6bc472e7a9..99ec5a1c1a 100755 --- a/tests/metrics/gha-run.sh +++ b/tests/metrics/gha-run.sh @@ -56,28 +56,24 @@ function make_tarball_results() { function run_test_launchtimes() { info "Running Launch Time test using ${KATA_HYPERVISOR} hypervisor" - create_symbolic_links ${KATA_HYPERVISOR} bash tests/metrics/time/launch_times.sh -i public.ecr.aws/ubuntu/ubuntu:latest -n 20 } function run_test_memory_usage() { info "Running memory-usage test using ${KATA_HYPERVISOR} hypervisor" - create_symbolic_links ${KATA_HYPERVISOR} bash tests/metrics/density/memory_usage.sh 20 5 } function run_test_memory_usage_inside_container() { info "Running memory-usage inside the container test using ${KATA_HYPERVISOR} hypervisor" - create_symbolic_links ${KATA_HYPERVISOR} bash tests/metrics/density/memory_usage_inside_container.sh 5 } function run_test_blogbench() { info "Running Blogbench test using ${KATA_HYPERVISOR} hypervisor" - create_symbolic_links ${KATA_HYPERVISOR} bash tests/metrics/storage/blogbench.sh check_metrics @@ -88,14 +84,13 @@ function run_test_tensorflow() { # ToDo: remove the exit once the metrics workflow is stable exit 0 - create_symbolic_links ${KATA_HYPERVISOR} bash tests/metrics/machine_learning/tensorflow.sh 1 20 } function main() { action="${1:-}" case "${action}" in - install-kata) install_kata ;; + install-kata) install_kata && install_checkmetrics ;; make-tarball-results) make_tarball_results ;; run-test-launchtimes) run_test_launchtimes ;; run-test-memory-usage) run_test_memory_usage ;; diff --git a/versions.yaml b/versions.yaml index e31dab1645..f316be8659 100644 --- a/versions.yaml +++ b/versions.yaml @@ -230,6 +230,9 @@ externals: description: "CLI tool for Container Runtime Interface (CRI)" url: "https://github.com/kubernetes-sigs/cri-tools" version: "1.23.0" + # As we don't want to disrupt what we have on the `tests` repo, let's + # create a "latest" entry and use that for the GitHub actions tests. + latest: "v1.27" cryptsetup: description: "A utility used to setup disk encryption, integrity protection"