diff --git a/.github/workflows/basic-ci-amd64.yaml b/.github/workflows/basic-ci-amd64.yaml index 72348b2454..02f66387d4 100644 --- a/.github/workflows/basic-ci-amd64.yaml +++ b/.github/workflows/basic-ci-amd64.yaml @@ -88,7 +88,7 @@ jobs: fail-fast: false matrix: containerd_version: ['lts', 'active'] - vmm: ['clh', 'clh-runtime-rs', 'dragonball', 'qemu', 'qemu-runtime-rs'] + vmm: ['clh', 'clh-runtime-rs', 'dragonball', 'qemu', 'qemu-runtime-rs', 'fc'] concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-run-containerd-stability-amd64-${{ toJSON(matrix) }} cancel-in-progress: true @@ -245,6 +245,7 @@ jobs: - qemu - clh-runtime-rs - qemu-runtime-rs + - fc concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-nerdctl-amd64-${{ toJSON(matrix) }} cancel-in-progress: true diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ac1a3234c6..bdd1dbdd57 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -403,11 +403,13 @@ jobs: { containerd_version: lts, vmm: qemu }, { containerd_version: lts, vmm: clh-runtime-rs }, { containerd_version: lts, vmm: qemu-runtime-rs }, + { containerd_version: lts, vmm: fc }, { containerd_version: active, vmm: clh }, { containerd_version: active, vmm: dragonball }, { containerd_version: active, vmm: qemu }, { containerd_version: active, vmm: clh-runtime-rs }, { containerd_version: active, vmm: qemu-runtime-rs }, + { containerd_version: active, vmm: fc }, ] concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-cri-amd64-${{ toJSON(matrix) }} diff --git a/.github/workflows/run-k8s-tests-on-free-runner.yaml b/.github/workflows/run-k8s-tests-on-free-runner.yaml index 9845bb308b..e6f73fb379 100644 --- a/.github/workflows/run-k8s-tests-on-free-runner.yaml +++ b/.github/workflows/run-k8s-tests-on-free-runner.yaml @@ -50,6 +50,7 @@ jobs: { vmm: qemu-runtime-rs, containerd_version: active }, { vmm: clh-runtime-rs, containerd_version: lts }, { vmm: clh-runtime-rs, containerd_version: active }, + { vmm: fc, containerd_version: active, snapshotter: blockfile }, ] concurrency: group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-free-runner-${{ toJSON(matrix) }} @@ -68,6 +69,7 @@ jobs: K8S_TEST_HOST_TYPE: baremetal-no-attestation CONTAINER_ENGINE: containerd CONTAINER_ENGINE_VERSION: ${{ matrix.environment.containerd_version }} + SNAPSHOTTER: ${{ matrix.environment.snapshotter || '' }} GH_TOKEN: ${{ github.token }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -115,6 +117,10 @@ jobs: - name: Install `bats` run: bash tests/integration/kubernetes/gha-run.sh install-bats + - name: Configure ${{ matrix.environment.snapshotter }} snapshotter + if: ${{ matrix.environment.snapshotter != '' }} + run: bash tests/integration/kubernetes/gha-run.sh configure-snapshotter + - name: Deploy Kata timeout-minutes: 20 run: bash tests/integration/kubernetes/gha-run.sh deploy-kata diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index d271e10b16..a266235545 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -38,7 +38,53 @@ export GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-oci-distribution}" export TARGET_ARCH="${TARGET_ARCH:-x86_64}" export RUNS_ON_AKS="${RUNS_ON_AKS:-false}" -function configure_devmapper() { +function _configure_containerd_snapshotter() { + local snapshotter_name="${1}" + local tomlq_filter="${2}" + + local containerd_config_file + case "${KUBERNETES}" in + k3s) + containerd_config_file="/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl" + sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml "${containerd_config_file}" + ;; + kubeadm|vanilla) + containerd_config_file="/etc/containerd/config.toml" + ;; + *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; + esac + + install_tomlq + + local containerd_arch + containerd_arch="$(uname -m)" + case "${containerd_arch}" in + x86_64) containerd_arch="amd64" ;; + aarch64|arm64) containerd_arch="arm64" ;; + esac + + echo "Updating containerd config with tomlq for ${snapshotter_name} snapshotter..." + local config_tmp_file + config_tmp_file="$(sudo mktemp)" + # shellcheck disable=SC2016 + sudo cat "${containerd_config_file}" | tomlq -t --arg platform "linux/${containerd_arch}" "${tomlq_filter}" | sudo tee "${config_tmp_file}" > /dev/null + sudo mv "${config_tmp_file}" "${containerd_config_file}" + + uninstall_tomlq + + case "${KUBERNETES}" in + k3s) sudo systemctl restart k3s ;; + vanilla|kubeadm) sudo systemctl restart containerd ;; + *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; + esac + + sleep 60s + sudo cat "${containerd_config_file}" + + info "${snapshotter_name} snapshotter configured" +} + +function _setup_devmapper_device() { sudo mkdir -p /var/lib/containerd/devmapper sudo truncate --size 10G /var/lib/containerd/devmapper/data-disk.img sudo truncate --size 10G /var/lib/containerd/devmapper/meta-disk.img @@ -62,75 +108,12 @@ EOF sudo systemctl daemon-reload sudo systemctl enable --now containerd-devmapper - # Time to setup the thin pool for consumption. - # The table arguments are such. - # start block in the virtual device - # length of the segment (block device size in bytes / Sector size (512) - # metadata device - # block data device - # data_block_size Currently set it 512 (128KB) - # low_water_mark. Copied this from containerd snapshotter test setup - # no. of feature arguments - # Skip zeroing blocks for new volumes. sudo dmsetup create contd-thin-pool \ --table "0 20971520 thin-pool /dev/loop21 /dev/loop20 512 32768 1 skip_block_zeroing" +} - case "${KUBERNETES}" in - k3s) - containerd_config_file="/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl" - sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml "${containerd_config_file}" - ;; - kubeadm) - containerd_config_file="/etc/containerd/config.toml" - ;; - *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; - esac - - # We need to use tomlq to update the containerd config with the devmapper configuration, - # as it's a more complex update that involves adding new entries and modifying existing ones - # for two different containerd versions. - install_tomlq - - containerd_arch="$(uname -m)" - case "${containerd_arch}" in - x86_64) containerd_arch="amd64" ;; - aarch64|arm64) containerd_arch="arm64" ;; - esac - - echo "Updating containerd config with tomlq..." - config_tmp_file="$(sudo mktemp)" - # shellcheck disable=SC2016 - sudo cat "${containerd_config_file}" | tomlq -t --arg platform "linux/${containerd_arch}" ' - .plugins["io.containerd.snapshotter.v1.devmapper"].pool_name = "contd-thin-pool" - | .plugins["io.containerd.snapshotter.v1.devmapper"].base_image_size = "4096MB" - | .plugins["io.containerd.transfer.v1.local"].unpack_config = - [((.plugins["io.containerd.transfer.v1.local"].unpack_config[0] // {}) + {platform: $platform, snapshotter: "devmapper"})] - | if .version == 3 then - .plugins["io.containerd.cri.v1.images"].snapshotter = "devmapper" - else - .plugins["io.containerd.grpc.v1.cri"].containerd.snapshotter = "devmapper" - end - ' | sudo tee "${config_tmp_file}" > /dev/null - sudo mv "${config_tmp_file}" "${containerd_config_file}" - - # We only need tomlq for this configuration. - # yq, installed by install_tomlq, might cause an issue with go-based yq used by CI. - # So we uninstall tomlq to remove the yq from PATH and avoid any potential conflict. - uninstall_tomlq - - case "${KUBERNETES}" in - k3s) - sudo systemctl restart k3s ;; - kubeadm) - sudo systemctl restart containerd ;; - *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; - esac - - sleep 60s - sudo cat "${containerd_config_file}" - - if [[ "${KUBERNETES}" = 'k3s' ]] - then +function _verify_devmapper() { + if [[ "${KUBERNETES}" = 'k3s' ]]; then local ctr_dm_status local result @@ -149,11 +132,76 @@ EOF sudo dmsetup status -v } +function _setup_blockfile_device() { + sudo mkdir -p /opt/containerd + sudo dd if=/dev/zero of=/opt/containerd/blockfile bs=1M count=500 + sudo mkfs.ext4 /opt/containerd/blockfile +} + +function _configure_fc_shared_fs() { + if is_firecracker_hypervisor; then + local fc_config="" + for candidate in \ + "/opt/kata/share/defaults/kata-containers/runtimes/fc/configuration-fc.toml" \ + "/opt/kata/share/defaults/kata-containers/configuration-fc.toml" \ + "/etc/kata-containers/configuration-fc.toml"; do + if [[ -f "${candidate}" ]]; then + fc_config="${candidate}" + break + fi + done + if [[ -z "${fc_config}" ]]; then + warn "FC kata config not found, shared_fs unchanged" + return 0 + fi + info "Setting shared_fs=none in ${fc_config}" + sudo sed -i 's/^#\?shared_fs.*/shared_fs = "none"/' "${fc_config}" + fi +} + +# shellcheck disable=SC2016 +readonly devmapper_tomlq_filter=' + .plugins["io.containerd.snapshotter.v1.devmapper"].pool_name = "contd-thin-pool" + | .plugins["io.containerd.snapshotter.v1.devmapper"].base_image_size = "4096MB" + | .plugins["io.containerd.transfer.v1.local"].unpack_config = + [((.plugins["io.containerd.transfer.v1.local"].unpack_config[0] // {}) + {platform: $platform, snapshotter: "devmapper"})] + | if .version == 3 then + .plugins["io.containerd.cri.v1.images"].snapshotter = "devmapper" + else + .plugins["io.containerd.grpc.v1.cri"].containerd.snapshotter = "devmapper" + end +' + +# shellcheck disable=SC2016 +readonly blockfile_tomlq_filter=' + .plugins["io.containerd.snapshotter.v1.blockfile"].root_path = "/var/lib/containerd-blockfile" + | .plugins["io.containerd.snapshotter.v1.blockfile"].scratch_file = "/opt/containerd/blockfile" + | .plugins["io.containerd.snapshotter.v1.blockfile"].fs_type = "ext4" + | .plugins["io.containerd.snapshotter.v1.blockfile"].mount_options = [] + | .plugins["io.containerd.snapshotter.v1.blockfile"].recreate_scratch = true + | .plugins["io.containerd.transfer.v1.local"].unpack_config = + [((.plugins["io.containerd.transfer.v1.local"].unpack_config[0] // {}) + {platform: $platform, snapshotter: "blockfile"})] + | if .version == 3 then + .plugins["io.containerd.cri.v1.images"].snapshotter = "blockfile" + else + .plugins["io.containerd.grpc.v1.cri"].containerd.snapshotter = "blockfile" + end +' + function configure_snapshotter() { echo "::group::Configuring ${SNAPSHOTTER}" case "${SNAPSHOTTER}" in - devmapper) configure_devmapper ;; + devmapper) + _setup_devmapper_device + _configure_containerd_snapshotter "devmapper" "${devmapper_tomlq_filter}" + _verify_devmapper + ;; + blockfile) + _setup_blockfile_device + _configure_containerd_snapshotter "blockfile" "${blockfile_tomlq_filter}" + _configure_fc_shared_fs + ;; *) >&2 echo "${SNAPSHOTTER} flavour is not supported"; exit 2 ;; esac diff --git a/tools/testing/gatekeeper/required-tests.yaml b/tools/testing/gatekeeper/required-tests.yaml index cc03e78629..d9b1a76010 100644 --- a/tools/testing/gatekeeper/required-tests.yaml +++ b/tools/testing/gatekeeper/required-tests.yaml @@ -49,16 +49,19 @@ mapping: - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (active, dragonball) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (active, qemu) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (active, qemu-runtime-rs) + - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (active, fc) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, clh) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, clh-runtime-rs) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, dragonball) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, qemu) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, qemu-runtime-rs) + - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, fc) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-kata-agent-apis - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nerdctl-tests (clh-runtime-rs) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nerdctl-tests (dragonball) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nerdctl-tests (qemu) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nerdctl-tests (qemu-runtime-rs) + - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nerdctl-tests (fc) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nydus (active, clh) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nydus (active, dragonball) - Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nydus (active, qemu) @@ -70,11 +73,13 @@ mapping: - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (active, dragonball) / run-cri-containerd-amd64 (active, dragonball) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (active, qemu-runtime-rs) / run-cri-containerd-amd64 (active, qemu-runtime-rs) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (active, qemu) / run-cri-containerd-amd64 (active, qemu) + - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (active, fc) / run-cri-containerd-amd64 (active, fc) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, clh) / run-cri-containerd-amd64 (lts, clh) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, clh-runtime-rs) / run-cri-containerd-amd64 (lts, clh-runtime-rs) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, dragonball) / run-cri-containerd-amd64 (lts, dragonball) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, qemu-runtime-rs) / run-cri-containerd-amd64 (lts, qemu-runtime-rs) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, qemu) / run-cri-containerd-amd64 (lts, qemu) + - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, fc) / run-cri-containerd-amd64 (lts, fc) - Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-s390x (active, qemu) / run-cri-containerd-s390x (active, qemu) #- Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-arm64 / run-k8s-tests-on-arm64 (qemu, kubeadm) - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (cbl-mariner, clh-azure, normal) @@ -92,6 +97,7 @@ mapping: - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-free-runner / run-k8s-tests (qemu-runtime-rs, active) - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-free-runner / run-k8s-tests (clh-runtime-rs, lts) - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-free-runner / run-k8s-tests (clh-runtime-rs, active) + - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-free-runner / run-k8s-tests (fc, active, blockfile) - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-zvsi / run-k8s-tests (devmapper, qemu, kubeadm) - Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-zvsi / run-k8s-tests (nydus, qemu-coco-dev, kubeadm) - Kata Containers CI / kata-containers-ci-on-push / run-kata-coco-tests / run-k8s-tests-on-tee (sev-snp, qemu-snp)