diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 78b2a49aed..d7245a8439 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -81,3 +81,10 @@ jobs: with: tarball-suffix: -${{ inputs.tag }} commit-hash: ${{ inputs.commit-hash }} + + run-nydus-tests: + needs: build-kata-static-tarball-amd64 + uses: ./.github/workflows/run-nydus-tests.yaml + with: + tarball-suffix: -${{ inputs.tag }} + commit-hash: ${{ inputs.commit-hash }} diff --git a/.github/workflows/run-nydus-tests.yaml b/.github/workflows/run-nydus-tests.yaml new file mode 100644 index 0000000000..647582c08e --- /dev/null +++ b/.github/workflows/run-nydus-tests.yaml @@ -0,0 +1,42 @@ +name: CI | Run nydus tests +on: + workflow_call: + inputs: + tarball-suffix: + required: false + type: string + commit-hash: + required: false + type: string + +jobs: + run-nydus: + strategy: + fail-fast: true + matrix: + containerd_version: ['lts', 'active'] + vmm: ['clh', 'qemu', 'dragonball'] + runs-on: garm-ubuntu-2204 + env: + CONTAINERD_VERSION: ${{ matrix.containerd_version }} + GOPATH: ${{ github.workspace }} + KATA_HYPERVISOR: ${{ matrix.vmm }} + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.commit-hash }} + + - name: Install dependencies + run: bash tests/integration/nydus/gha-run.sh install-dependencies + + - name: get-kata-tarball + uses: actions/download-artifact@v3 + with: + name: kata-static-tarball-amd64${{ inputs.tarball-suffix }} + path: kata-artifacts + + - name: Install kata + run: bash tests/integration/nydus/gha-run.sh install-kata kata-artifacts + + - name: Run nydus tests + run: bash tests/integration/nydus/gha-run.sh run diff --git a/tests/integration/nydus/gha-run.sh b/tests/integration/nydus/gha-run.sh new file mode 100755 index 0000000000..80ce8dcc3a --- /dev/null +++ b/tests/integration/nydus/gha-run.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail + +kata_tarball_dir="${2:-kata-artifacts}" +nydus_dir="$(dirname "$(readlink -f "$0")")" +source "${cri_containerd_dir}/../../common.bash" + +function install_dependencies() { + info "Installing the dependencies needed for running the nydus tests" + + return 0 +} + +function run() { + info "Running nydus tests using ${KATA_HYPERVISOR} hypervisor" + + return 0 +} + +function main() { + action="${1:-}" + case "${action}" in + install-dependencies) install_dependencies ;; + install-kata) return 0 ;; + run) run ;; + *) >&2 die "Invalid argument" ;; + esac +} + +main "$@" diff --git a/tests/integration/nydus/nydus-container.yaml b/tests/integration/nydus/nydus-container.yaml new file mode 100644 index 0000000000..a30327adee --- /dev/null +++ b/tests/integration/nydus/nydus-container.yaml @@ -0,0 +1,12 @@ +metadata: + name: nydus-container +image: + image: ghcr.io/dragonflyoss/image-service/alpine:nydus-latest +command: +- tail +- -f +- /dev/null +linux: + resources: + memory_limit_in_bytes: 524288000 +log_path: nydus.0.log diff --git a/tests/integration/nydus/nydus-sandbox.yaml b/tests/integration/nydus/nydus-sandbox.yaml new file mode 100644 index 0000000000..9f039d726d --- /dev/null +++ b/tests/integration/nydus/nydus-sandbox.yaml @@ -0,0 +1,5 @@ +metadata: + attempt: 1 + name: nydus-sandbox + namespace: default +log_directory: /tmp diff --git a/tests/integration/nydus/nydus_tests.sh b/tests/integration/nydus/nydus_tests.sh new file mode 100755 index 0000000000..fe49580da4 --- /dev/null +++ b/tests/integration/nydus/nydus_tests.sh @@ -0,0 +1,211 @@ +#!/bin/bash +# +# Copyright (c) 2022 Ant Group +# +# SPDX-License-Identifier: Apache-2.0 +# +# This will test the nydus feature is working properly + +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +dir_path=$(dirname "$0") +source "${dir_path}/../../lib/common.bash" +source "${dir_path}/../../.ci/lib.sh" +source "/etc/os-release" || source "/usr/lib/os-release" +KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" + +need_restore_kata_config=false +kata_config_backup="/tmp/kata-configuration.toml" +SYSCONFIG_FILE="/etc/kata-containers/configuration.toml" +DEFAULT_CONFIG_FILE="/opt/kata/share/defaults/kata-containers/configuration-qemu.toml" +CLH_CONFIG_FILE="/opt/kata/share/defaults/kata-containers/configuration-clh.toml" +DB_CONFIG_FILE="/opt/kata/share/defaults/kata-containers/configuration-dragonball.toml" +need_restore_containerd_config=false +containerd_config="/etc/containerd/config.toml" +containerd_config_backup="/tmp/containerd.config.toml" + +# test image for container +IMAGE="${IMAGE:-ghcr.io/dragonflyoss/image-service/alpine:nydus-latest}" + +if [ "$KATA_HYPERVISOR" != "qemu" ] && [ "$KATA_HYPERVISOR" != "cloud-hypervisor" ] && [ "$KATA_HYPERVISOR" != "dragonball" ]; then + echo "Skip nydus test for $KATA_HYPERVISOR, it only works for QEMU/CLH/DB now." + exit 0 +fi + +arch="$(uname -m)" +if [ "$arch" != "x86_64" ]; then + echo "Skip nydus test for $arch, it only works for x86_64 now. See https://github.com/kata-containers/tests/issues/4445" + exit 0 +fi + +function install_from_tarball() { + local package_name="$1" + local binary_name="$2" + [ -n "$package_name" ] || die "need package_name" + [ -n "$binary_name" ] || die "need package release binary_name" + + local url=$(get_version "externals.${package_name}.url") + local version=$(get_version "externals.${package_name}.version") + local tarball_url="${url}/releases/download/${version}/${binary_name}-${version}-$arch.tgz" + if [ "${package_name}" == "nydus" ]; then + local goarch="$(${dir_path}/../../.ci/kata-arch.sh --golang)" + tarball_url="${url}/releases/download/${version}/${binary_name}-${version}-linux-$goarch.tgz" + fi + echo "Download tarball from ${tarball_url}" + curl -Ls "$tarball_url" | sudo tar xfz - -C /usr/local/bin --strip-components=1 +} + +function setup_nydus() { + # install nydus + install_from_tarball "nydus" "nydus-static" + + # install nydus-snapshotter + install_from_tarball "nydus-snapshotter" "nydus-snapshotter" + + # Config nydus snapshotter + sudo -E cp "$dir_path/nydusd-config.json" /etc/ + + # start nydus-snapshotter + nohup /usr/local/bin/containerd-nydus-grpc \ + --config-path /etc/nydusd-config.json \ + --shared-daemon \ + --log-level debug \ + --root /var/lib/containerd/io.containerd.snapshotter.v1.nydus \ + --cache-dir /var/lib/nydus/cache \ + --nydusd-path /usr/local/bin/nydusd \ + --nydusimg-path /usr/local/bin/nydus-image \ + --disable-cache-manager true \ + --enable-nydus-overlayfs true \ + --log-to-stdout >/dev/null 2>&1 & +} + +function config_kata() { + sudo mkdir -p /etc/kata-containers + if [ -f "$SYSCONFIG_FILE" ]; then + need_restore_kata_config=true + sudo cp -a "${SYSCONFIG_FILE}" "${kata_config_backup}" + elif [ "$KATA_HYPERVISOR" == "qemu" ]; then + sudo cp -a "${DEFAULT_CONFIG_FILE}" "${SYSCONFIG_FILE}" + elif [ "$KATA_HYPERVISOR" == "dragonball" ]; then + sudo cp -a "${DB_CONFIG_FILE}" "${SYSCONFIG_FILE}" + else + sudo cp -a "${CLH_CONFIG_FILE}" "${SYSCONFIG_FILE}" + fi + + echo "Enabling all debug options in file ${SYSCONFIG_FILE}" + sudo sed -i -e 's/^#\(enable_debug\).*=.*$/\1 = true/g' "${SYSCONFIG_FILE}" + sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.log=debug"/g' "${SYSCONFIG_FILE}" + + if [ "$KATA_HYPERVISOR" != "dragonball" ]; then + sudo sed -i 's|^shared_fs.*|shared_fs = "virtio-fs-nydus"|g' "${SYSCONFIG_FILE}" + sudo sed -i 's|^virtio_fs_daemon.*|virtio_fs_daemon = "/usr/local/bin/nydusd"|g' "${SYSCONFIG_FILE}" + fi + + sudo sed -i 's|^virtio_fs_extra_args.*|virtio_fs_extra_args = []|g' "${SYSCONFIG_FILE}" +} + +function config_containerd() { + readonly runc_path=$(command -v runc) + sudo mkdir -p /etc/containerd/ + if [ -f "$containerd_config" ]; then + need_restore_containerd_config=true + sudo cp -a "${containerd_config}" "${containerd_config_backup}" + else + sudo rm "${containerd_config}" + fi + + cat <