mirror of
https://github.com/kata-containers/kata-containers.git
synced 2026-07-10 07:05:51 +00:00
ci: add Firecracker testing (k8s with blockfile snapshotter)
Firecracker currently has no testing CI, only build CI. Add fc hypervisor to the test matrices for stability, CRI containerd, nerdctl, and Kubernetes tests. For the Kubernetes test matrix, configure the blockfile snapshotter instead of devmapper as devmapper requires 10G+ loop devices which is too space costly. The blockfile snapshotter uses a 500MB ext4 scratch file. Firecracker requires shared_fs=none in the kata configuration when using block-based rootfs; this is set by the _configure_fc_shared_fs helper which searches all three possible FC config paths and warns if none is found. The other test matrices (stability, CRI containerd, nerdctl) run FC with the default overlayfs snapshotter, which validates basic FC functionality (boot, agent, container lifecycle) without requiring snapshotter pre-configuration in those workflows. Refactor configure_snapshotter to extract the shared containerd configuration flow (config path resolution, tomlq install/uninstall, arch mapping, config update, service restart) into a reusable _configure_containerd_snapshotter helper so that adding new snapshotters does not require duplicating the entire function. Related #7996, #7872 Signed-off-by: marvelshan <reborn7875@gmail.com>
This commit is contained in:
marvelshan
committed by
Fabiano Fidêncio
Fabiano Fidêncio
parent
9e65f85ccd
commit
516d62d717
3
.github/workflows/basic-ci-amd64.yaml
vendored
3
.github/workflows/basic-ci-amd64.yaml
vendored
@@ -88,7 +88,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
containerd_version: ['lts', 'active']
|
||||
vmm: ['clh', 'clh-runtime-rs', 'dragonball', 'qemu', 'qemu-runtime-rs']
|
||||
vmm: ['clh', 'clh-runtime-rs', 'dragonball', 'qemu', 'qemu-runtime-rs', 'fc']
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-run-containerd-stability-amd64-${{ toJSON(matrix) }}
|
||||
cancel-in-progress: true
|
||||
@@ -245,6 +245,7 @@ jobs:
|
||||
- qemu
|
||||
- clh-runtime-rs
|
||||
- qemu-runtime-rs
|
||||
- fc
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-nerdctl-amd64-${{ toJSON(matrix) }}
|
||||
cancel-in-progress: true
|
||||
|
||||
2
.github/workflows/ci.yaml
vendored
2
.github/workflows/ci.yaml
vendored
@@ -403,11 +403,13 @@ jobs:
|
||||
{ containerd_version: lts, vmm: qemu },
|
||||
{ containerd_version: lts, vmm: clh-runtime-rs },
|
||||
{ containerd_version: lts, vmm: qemu-runtime-rs },
|
||||
{ containerd_version: lts, vmm: fc },
|
||||
{ containerd_version: active, vmm: clh },
|
||||
{ containerd_version: active, vmm: dragonball },
|
||||
{ containerd_version: active, vmm: qemu },
|
||||
{ containerd_version: active, vmm: clh-runtime-rs },
|
||||
{ containerd_version: active, vmm: qemu-runtime-rs },
|
||||
{ containerd_version: active, vmm: fc },
|
||||
]
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-cri-amd64-${{ toJSON(matrix) }}
|
||||
|
||||
@@ -50,6 +50,7 @@ jobs:
|
||||
{ vmm: qemu-runtime-rs, containerd_version: active },
|
||||
{ vmm: clh-runtime-rs, containerd_version: lts },
|
||||
{ vmm: clh-runtime-rs, containerd_version: active },
|
||||
{ vmm: fc, containerd_version: active, snapshotter: blockfile },
|
||||
]
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.job }}-${{ github.event.pull_request.number || github.ref }}-free-runner-${{ toJSON(matrix) }}
|
||||
@@ -68,6 +69,7 @@ jobs:
|
||||
K8S_TEST_HOST_TYPE: baremetal-no-attestation
|
||||
CONTAINER_ENGINE: containerd
|
||||
CONTAINER_ENGINE_VERSION: ${{ matrix.environment.containerd_version }}
|
||||
SNAPSHOTTER: ${{ matrix.environment.snapshotter || '' }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
@@ -115,6 +117,10 @@ jobs:
|
||||
- name: Install `bats`
|
||||
run: bash tests/integration/kubernetes/gha-run.sh install-bats
|
||||
|
||||
- name: Configure ${{ matrix.environment.snapshotter }} snapshotter
|
||||
if: ${{ matrix.environment.snapshotter != '' }}
|
||||
run: bash tests/integration/kubernetes/gha-run.sh configure-snapshotter
|
||||
|
||||
- name: Deploy Kata
|
||||
timeout-minutes: 20
|
||||
run: bash tests/integration/kubernetes/gha-run.sh deploy-kata
|
||||
|
||||
@@ -38,7 +38,53 @@ export GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-oci-distribution}"
|
||||
export TARGET_ARCH="${TARGET_ARCH:-x86_64}"
|
||||
export RUNS_ON_AKS="${RUNS_ON_AKS:-false}"
|
||||
|
||||
function configure_devmapper() {
|
||||
function _configure_containerd_snapshotter() {
|
||||
local snapshotter_name="${1}"
|
||||
local tomlq_filter="${2}"
|
||||
|
||||
local containerd_config_file
|
||||
case "${KUBERNETES}" in
|
||||
k3s)
|
||||
containerd_config_file="/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl"
|
||||
sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml "${containerd_config_file}"
|
||||
;;
|
||||
kubeadm|vanilla)
|
||||
containerd_config_file="/etc/containerd/config.toml"
|
||||
;;
|
||||
*) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;;
|
||||
esac
|
||||
|
||||
install_tomlq
|
||||
|
||||
local containerd_arch
|
||||
containerd_arch="$(uname -m)"
|
||||
case "${containerd_arch}" in
|
||||
x86_64) containerd_arch="amd64" ;;
|
||||
aarch64|arm64) containerd_arch="arm64" ;;
|
||||
esac
|
||||
|
||||
echo "Updating containerd config with tomlq for ${snapshotter_name} snapshotter..."
|
||||
local config_tmp_file
|
||||
config_tmp_file="$(sudo mktemp)"
|
||||
# shellcheck disable=SC2016
|
||||
sudo cat "${containerd_config_file}" | tomlq -t --arg platform "linux/${containerd_arch}" "${tomlq_filter}" | sudo tee "${config_tmp_file}" > /dev/null
|
||||
sudo mv "${config_tmp_file}" "${containerd_config_file}"
|
||||
|
||||
uninstall_tomlq
|
||||
|
||||
case "${KUBERNETES}" in
|
||||
k3s) sudo systemctl restart k3s ;;
|
||||
vanilla|kubeadm) sudo systemctl restart containerd ;;
|
||||
*) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;;
|
||||
esac
|
||||
|
||||
sleep 60s
|
||||
sudo cat "${containerd_config_file}"
|
||||
|
||||
info "${snapshotter_name} snapshotter configured"
|
||||
}
|
||||
|
||||
function _setup_devmapper_device() {
|
||||
sudo mkdir -p /var/lib/containerd/devmapper
|
||||
sudo truncate --size 10G /var/lib/containerd/devmapper/data-disk.img
|
||||
sudo truncate --size 10G /var/lib/containerd/devmapper/meta-disk.img
|
||||
@@ -62,75 +108,12 @@ EOF
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now containerd-devmapper
|
||||
|
||||
# Time to setup the thin pool for consumption.
|
||||
# The table arguments are such.
|
||||
# start block in the virtual device
|
||||
# length of the segment (block device size in bytes / Sector size (512)
|
||||
# metadata device
|
||||
# block data device
|
||||
# data_block_size Currently set it 512 (128KB)
|
||||
# low_water_mark. Copied this from containerd snapshotter test setup
|
||||
# no. of feature arguments
|
||||
# Skip zeroing blocks for new volumes.
|
||||
sudo dmsetup create contd-thin-pool \
|
||||
--table "0 20971520 thin-pool /dev/loop21 /dev/loop20 512 32768 1 skip_block_zeroing"
|
||||
}
|
||||
|
||||
case "${KUBERNETES}" in
|
||||
k3s)
|
||||
containerd_config_file="/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl"
|
||||
sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml "${containerd_config_file}"
|
||||
;;
|
||||
kubeadm)
|
||||
containerd_config_file="/etc/containerd/config.toml"
|
||||
;;
|
||||
*) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;;
|
||||
esac
|
||||
|
||||
# We need to use tomlq to update the containerd config with the devmapper configuration,
|
||||
# as it's a more complex update that involves adding new entries and modifying existing ones
|
||||
# for two different containerd versions.
|
||||
install_tomlq
|
||||
|
||||
containerd_arch="$(uname -m)"
|
||||
case "${containerd_arch}" in
|
||||
x86_64) containerd_arch="amd64" ;;
|
||||
aarch64|arm64) containerd_arch="arm64" ;;
|
||||
esac
|
||||
|
||||
echo "Updating containerd config with tomlq..."
|
||||
config_tmp_file="$(sudo mktemp)"
|
||||
# shellcheck disable=SC2016
|
||||
sudo cat "${containerd_config_file}" | tomlq -t --arg platform "linux/${containerd_arch}" '
|
||||
.plugins["io.containerd.snapshotter.v1.devmapper"].pool_name = "contd-thin-pool"
|
||||
| .plugins["io.containerd.snapshotter.v1.devmapper"].base_image_size = "4096MB"
|
||||
| .plugins["io.containerd.transfer.v1.local"].unpack_config =
|
||||
[((.plugins["io.containerd.transfer.v1.local"].unpack_config[0] // {}) + {platform: $platform, snapshotter: "devmapper"})]
|
||||
| if .version == 3 then
|
||||
.plugins["io.containerd.cri.v1.images"].snapshotter = "devmapper"
|
||||
else
|
||||
.plugins["io.containerd.grpc.v1.cri"].containerd.snapshotter = "devmapper"
|
||||
end
|
||||
' | sudo tee "${config_tmp_file}" > /dev/null
|
||||
sudo mv "${config_tmp_file}" "${containerd_config_file}"
|
||||
|
||||
# We only need tomlq for this configuration.
|
||||
# yq, installed by install_tomlq, might cause an issue with go-based yq used by CI.
|
||||
# So we uninstall tomlq to remove the yq from PATH and avoid any potential conflict.
|
||||
uninstall_tomlq
|
||||
|
||||
case "${KUBERNETES}" in
|
||||
k3s)
|
||||
sudo systemctl restart k3s ;;
|
||||
kubeadm)
|
||||
sudo systemctl restart containerd ;;
|
||||
*) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;;
|
||||
esac
|
||||
|
||||
sleep 60s
|
||||
sudo cat "${containerd_config_file}"
|
||||
|
||||
if [[ "${KUBERNETES}" = 'k3s' ]]
|
||||
then
|
||||
function _verify_devmapper() {
|
||||
if [[ "${KUBERNETES}" = 'k3s' ]]; then
|
||||
local ctr_dm_status
|
||||
local result
|
||||
|
||||
@@ -149,11 +132,76 @@ EOF
|
||||
sudo dmsetup status -v
|
||||
}
|
||||
|
||||
function _setup_blockfile_device() {
|
||||
sudo mkdir -p /opt/containerd
|
||||
sudo dd if=/dev/zero of=/opt/containerd/blockfile bs=1M count=500
|
||||
sudo mkfs.ext4 /opt/containerd/blockfile
|
||||
}
|
||||
|
||||
function _configure_fc_shared_fs() {
|
||||
if is_firecracker_hypervisor; then
|
||||
local fc_config=""
|
||||
for candidate in \
|
||||
"/opt/kata/share/defaults/kata-containers/runtimes/fc/configuration-fc.toml" \
|
||||
"/opt/kata/share/defaults/kata-containers/configuration-fc.toml" \
|
||||
"/etc/kata-containers/configuration-fc.toml"; do
|
||||
if [[ -f "${candidate}" ]]; then
|
||||
fc_config="${candidate}"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ -z "${fc_config}" ]]; then
|
||||
warn "FC kata config not found, shared_fs unchanged"
|
||||
return 0
|
||||
fi
|
||||
info "Setting shared_fs=none in ${fc_config}"
|
||||
sudo sed -i 's/^#\?shared_fs.*/shared_fs = "none"/' "${fc_config}"
|
||||
fi
|
||||
}
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
readonly devmapper_tomlq_filter='
|
||||
.plugins["io.containerd.snapshotter.v1.devmapper"].pool_name = "contd-thin-pool"
|
||||
| .plugins["io.containerd.snapshotter.v1.devmapper"].base_image_size = "4096MB"
|
||||
| .plugins["io.containerd.transfer.v1.local"].unpack_config =
|
||||
[((.plugins["io.containerd.transfer.v1.local"].unpack_config[0] // {}) + {platform: $platform, snapshotter: "devmapper"})]
|
||||
| if .version == 3 then
|
||||
.plugins["io.containerd.cri.v1.images"].snapshotter = "devmapper"
|
||||
else
|
||||
.plugins["io.containerd.grpc.v1.cri"].containerd.snapshotter = "devmapper"
|
||||
end
|
||||
'
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
readonly blockfile_tomlq_filter='
|
||||
.plugins["io.containerd.snapshotter.v1.blockfile"].root_path = "/var/lib/containerd-blockfile"
|
||||
| .plugins["io.containerd.snapshotter.v1.blockfile"].scratch_file = "/opt/containerd/blockfile"
|
||||
| .plugins["io.containerd.snapshotter.v1.blockfile"].fs_type = "ext4"
|
||||
| .plugins["io.containerd.snapshotter.v1.blockfile"].mount_options = []
|
||||
| .plugins["io.containerd.snapshotter.v1.blockfile"].recreate_scratch = true
|
||||
| .plugins["io.containerd.transfer.v1.local"].unpack_config =
|
||||
[((.plugins["io.containerd.transfer.v1.local"].unpack_config[0] // {}) + {platform: $platform, snapshotter: "blockfile"})]
|
||||
| if .version == 3 then
|
||||
.plugins["io.containerd.cri.v1.images"].snapshotter = "blockfile"
|
||||
else
|
||||
.plugins["io.containerd.grpc.v1.cri"].containerd.snapshotter = "blockfile"
|
||||
end
|
||||
'
|
||||
|
||||
function configure_snapshotter() {
|
||||
echo "::group::Configuring ${SNAPSHOTTER}"
|
||||
|
||||
case "${SNAPSHOTTER}" in
|
||||
devmapper) configure_devmapper ;;
|
||||
devmapper)
|
||||
_setup_devmapper_device
|
||||
_configure_containerd_snapshotter "devmapper" "${devmapper_tomlq_filter}"
|
||||
_verify_devmapper
|
||||
;;
|
||||
blockfile)
|
||||
_setup_blockfile_device
|
||||
_configure_containerd_snapshotter "blockfile" "${blockfile_tomlq_filter}"
|
||||
_configure_fc_shared_fs
|
||||
;;
|
||||
*) >&2 echo "${SNAPSHOTTER} flavour is not supported"; exit 2 ;;
|
||||
esac
|
||||
|
||||
|
||||
@@ -49,16 +49,19 @@ mapping:
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (active, dragonball)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (active, qemu)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (active, qemu-runtime-rs)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (active, fc)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, clh)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, clh-runtime-rs)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, dragonball)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, qemu)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, qemu-runtime-rs)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-containerd-stability (lts, fc)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-kata-agent-apis
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nerdctl-tests (clh-runtime-rs)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nerdctl-tests (dragonball)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nerdctl-tests (qemu)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nerdctl-tests (qemu-runtime-rs)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nerdctl-tests (fc)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nydus (active, clh)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nydus (active, dragonball)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-basic-amd64-tests / run-nydus (active, qemu)
|
||||
@@ -70,11 +73,13 @@ mapping:
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (active, dragonball) / run-cri-containerd-amd64 (active, dragonball)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (active, qemu-runtime-rs) / run-cri-containerd-amd64 (active, qemu-runtime-rs)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (active, qemu) / run-cri-containerd-amd64 (active, qemu)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (active, fc) / run-cri-containerd-amd64 (active, fc)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, clh) / run-cri-containerd-amd64 (lts, clh)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, clh-runtime-rs) / run-cri-containerd-amd64 (lts, clh-runtime-rs)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, dragonball) / run-cri-containerd-amd64 (lts, dragonball)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, qemu-runtime-rs) / run-cri-containerd-amd64 (lts, qemu-runtime-rs)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, qemu) / run-cri-containerd-amd64 (lts, qemu)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-amd64 (lts, fc) / run-cri-containerd-amd64 (lts, fc)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-cri-containerd-tests-s390x (active, qemu) / run-cri-containerd-s390x (active, qemu)
|
||||
#- Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-arm64 / run-k8s-tests-on-arm64 (qemu, kubeadm)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-aks / run-k8s-tests (cbl-mariner, clh-azure, normal)
|
||||
@@ -92,6 +97,7 @@ mapping:
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-free-runner / run-k8s-tests (qemu-runtime-rs, active)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-free-runner / run-k8s-tests (clh-runtime-rs, lts)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-free-runner / run-k8s-tests (clh-runtime-rs, active)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-free-runner / run-k8s-tests (fc, active, blockfile)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-zvsi / run-k8s-tests (devmapper, qemu, kubeadm)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-k8s-tests-on-zvsi / run-k8s-tests (nydus, qemu-coco-dev, kubeadm)
|
||||
- Kata Containers CI / kata-containers-ci-on-push / run-kata-coco-tests / run-k8s-tests-on-tee (sev-snp, qemu-snp)
|
||||
|
||||
Reference in New Issue
Block a user