diff --git a/.github/workflows/static-checks-dragonball.yaml b/.github/workflows/static-checks-dragonball.yaml deleted file mode 100644 index 2c99210a1..000000000 --- a/.github/workflows/static-checks-dragonball.yaml +++ /dev/null @@ -1,41 +0,0 @@ -on: - pull_request: - types: - - opened - - edited - - reopened - - synchronize - paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -name: Static checks dragonball -jobs: - test-dragonball: - runs-on: garm-ubuntu-2004 - env: - RUST_BACKTRACE: "1" - steps: - - uses: actions/checkout@v3 - - name: Set env - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends build-essential haveged - - name: Install Rust - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - ./ci/install_rust.sh - echo PATH="$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV - - name: Run Unit Test - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - cd src/dragonball - cargo version - rustc --version - sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test diff --git a/.github/workflows/static-checks.yaml b/.github/workflows/static-checks.yaml index cb113bfb0..ad2d2b7a4 100644 --- a/.github/workflows/static-checks.yaml +++ b/.github/workflows/static-checks.yaml @@ -12,84 +12,183 @@ concurrency: name: Static checks jobs: - static-checks: - runs-on: garm-ubuntu-2004 + check-kernel-config-version: + runs-on: ubuntu-latest + steps: + - name: Checkout the code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Ensure the kernel config version has been updated + run: | + kernel_dir="tools/packaging/kernel/" + kernel_version_file="${kernel_dir}kata_config_version" + modified_files=$(git diff --name-only origin/$GITHUB_BASE_REF..HEAD) + if git diff --name-only origin/$GITHUB_BASE_REF..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then + echo "Kernel directory has changed, checking if $kernel_version_file has been updated" + if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then + echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1) + else + echo "Readme file changed, no need for kernel config version update." + fi + echo "Check passed" + fi + + build-checks: + runs-on: ubuntu-20.04 strategy: - # We can set this to true whenever we're 100% sure that - # the all the tests are not flaky, otherwise we'll fail - # all the tests due to a single flaky instance. fail-fast: false matrix: - cmd: + component: + - agent + - dragonball + - runtime + - runtime-rs + - agent-ctl + - kata-ctl + - log-parser-rs + - runk + - trace-forwarder + command: - "make vendor" - - "make static-checks" - "make check" - "make test" - "sudo -E PATH=\"$PATH\" make test" + include: + - component: agent + component-path: src/agent + - component: dragonball + component-path: src/dragonball + - component: runtime + component-path: src/runtime + - component: runtime-rs + component-path: src/runtime-rs + - component: agent-ctl + component-path: src/tools/agent-ctl + - component: kata-ctl + component-path: src/tools/kata-ctl + - component: log-parser-rs + component-path: src/tools/log-parser-rs + - component: runk + component-path: src/tools/runk + - component: trace-forwarder + component-path: src/tools/trace-forwarder + - install-libseccomp: no + - component: agent + install-libseccomp: yes + - component: runk + install-libseccomp: yes + steps: + - name: Checkout the code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install yq + run: | + ./ci/install_yq.sh + env: + INSTALL_IN_GOPATH: false + - name: Install golang + if: ${{ matrix.component == 'runtime' }} + run: | + ./tests/install_go.sh -f -p + echo "/usr/local/go/bin" >> $GITHUB_PATH + - name: Install rust + if: ${{ matrix.component != 'runtime' }} + run: | + ./tests/install_rust.sh + echo "${HOME}/.cargo/bin" >> $GITHUB_PATH + - name: Install musl-tools + if: ${{ matrix.component != 'runtime' }} + run: sudo apt-get -y install musl-tools + - name: Install libseccomp + if: ${{ matrix.command != 'make vendor' && matrix.command != 'make check' && matrix.install-libseccomp == 'yes' }} + run: | + libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX) + gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX) + ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}" + echo "Set environment variables for the libseccomp crate to link the libseccomp library statically" + echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV + echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV + - name: Setup XDG_RUNTIME_DIR for the `runtime` tests + if: ${{ matrix.command != 'make vendor' && matrix.command != 'make check' && matrix.component == 'runtime' }} + run: | + XDG_RUNTIME_DIR=$(mktemp -d /tmp/kata-tests-$USER.XXX | tee >(xargs chmod 0700)) + echo "XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR}" >> $GITHUB_ENV + - name: Running `${{ matrix.command }}` for ${{ matrix.component }} + run: | + cd ${{ matrix.component-path }} + ${{ matrix.command }} + env: + RUST_BACKTRACE: "1" + + build-checks-depending-on-kvm: + runs-on: garm-ubuntu-2004-smaller + strategy: + fail-fast: false + matrix: + component: + - runtime-rs + include: + - component: runtime-rs + command: "sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test" + - component: runtime-rs + component-path: src/dragonball + steps: + - name: Checkout the code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system deps + run: | + sudo apt-get install -y build-essential musl-tools + - name: Install yq + run: | + sudo -E ./ci/install_yq.sh + env: + INSTALL_IN_GOPATH: false + - name: Install rust + run: | + export PATH="$PATH:/usr/local/bin" + ./tests/install_rust.sh + - name: Running `${{ matrix.command }}` for ${{ matrix.component }} + run: | + export PATH="$PATH:${HOME}/.cargo/bin" + cd ${{ matrix.component-path }} + ${{ matrix.command }} + env: + RUST_BACKTRACE: "1" + + static-checks: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + cmd: + - "make static-checks" env: - RUST_BACKTRACE: "1" - target_branch: ${{ github.base_ref }} GOPATH: ${{ github.workspace }} steps: - - name: Free disk space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - path: ./src/github.com/${{ github.repository }} - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends build-essential haveged - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: 1.19.3 - - name: Check kernel config version - run: | - cd "${{ github.workspace }}/src/github.com/${{ github.repository }}" - kernel_dir="tools/packaging/kernel/" - kernel_version_file="${kernel_dir}kata_config_version" - modified_files=$(git diff --name-only origin/main..HEAD) - if git diff --name-only origin/main..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then - echo "Kernel directory has changed, checking if $kernel_version_file has been updated" - if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then - echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1) - else - echo "Readme file changed, no need for kernel config version update." - fi - echo "Check passed" - fi - - name: Set PATH - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - echo "${{ github.workspace }}/bin" >> $GITHUB_PATH - - name: Setup - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh - - name: Installing rust - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh - PATH=$PATH:"$HOME/.cargo/bin" - rustup target add x86_64-unknown-linux-musl - rustup component add rustfmt clippy - - name: Setup seccomp - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX) - gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX) - cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}" - echo "Set environment variables for the libseccomp crate to link the libseccomp library statically" - echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV - echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV - - name: Run check - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - export PATH=$PATH:"$HOME/.cargo/bin" - export XDG_RUNTIME_DIR=$(mktemp -d /tmp/kata-tests-$USER.XXX | tee >(xargs chmod 0700)) - cd ${GOPATH}/src/github.com/${{ github.repository }} && ${{ matrix.cmd }} + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: ./src/github.com/${{ github.repository }} + - name: Install yq + run: | + cd ${GOPATH}/src/github.com/${{ github.repository }} + ./ci/install_yq.sh + env: + INSTALL_IN_GOPATH: false + - name: Install golang + run: | + cd ${GOPATH}/src/github.com/${{ github.repository }} + ./tests/install_go.sh -f -p + echo "/usr/local/go/bin" >> $GITHUB_PATH + - name: Install system dependencies + run: | + sudo apt-get -y install moreutils + - name: Run check + run: | + export PATH=${PATH}:${GOPATH}/bin + cd ${GOPATH}/src/github.com/${{ github.repository }} && ${{ matrix.cmd }} diff --git a/ci/install_libseccomp.sh b/ci/install_libseccomp.sh index 683d0f65b..5d53be733 100755 --- a/ci/install_libseccomp.sh +++ b/ci/install_libseccomp.sh @@ -7,12 +7,10 @@ set -o errexit -cidir=$(dirname "$0") -source "${cidir}/lib.sh" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +script_name="$(basename "${BASH_SOURCE[0]}")" -clone_tests_repo - -source "${tests_repo_dir}/.ci/lib.sh" +source "${script_dir}/../tests/common.bash" # The following variables if set on the environment will change the behavior # of gperf and libseccomp configure scripts, that may lead this script to @@ -25,11 +23,11 @@ workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)" # Variables for libseccomp libseccomp_version="${LIBSECCOMP_VERSION:-""}" if [ -z "${libseccomp_version}" ]; then - libseccomp_version=$(get_version "externals.libseccomp.version") + libseccomp_version=$(get_from_kata_deps "externals.libseccomp.version") fi libseccomp_url="${LIBSECCOMP_URL:-""}" if [ -z "${libseccomp_url}" ]; then - libseccomp_url=$(get_version "externals.libseccomp.url") + libseccomp_url=$(get_from_kata_deps "externals.libseccomp.url") fi libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz" libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}" @@ -38,11 +36,11 @@ cflags="-O2" # Variables for gperf gperf_version="${GPERF_VERSION:-""}" if [ -z "${gperf_version}" ]; then - gperf_version=$(get_version "externals.gperf.version") + gperf_version=$(get_from_kata_deps "externals.gperf.version") fi gperf_url="${GPERF_URL:-""}" if [ -z "${gperf_url}" ]; then - gperf_url=$(get_version "externals.gperf.url") + gperf_url=$(get_from_kata_deps "externals.gperf.url") fi gperf_tarball="gperf-${gperf_version}.tar.gz" gperf_tarball_url="${gperf_url}/${gperf_tarball}" diff --git a/src/agent/Makefile b/src/agent/Makefile index d058caf64..699b71ce1 100644 --- a/src/agent/Makefile +++ b/src/agent/Makefile @@ -148,7 +148,7 @@ vendor: #TARGET test: run cargo tests -test: +test: $(GENERATED_FILES) @cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture ##TARGET check: run test diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 8dd730ac8..abc96a553 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -49,7 +49,7 @@ else ##TARGET default: build code default: runtime show-header ##TARGET test: run cargo tests -test: +test: static-checks-build @cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture install: install-runtime install-configs endif diff --git a/src/tools/kata-ctl/Cargo.lock b/src/tools/kata-ctl/Cargo.lock index eda34fb17..2ddbcdbb9 100644 --- a/src/tools/kata-ctl/Cargo.lock +++ b/src/tools/kata-ctl/Cargo.lock @@ -1946,6 +1946,7 @@ dependencies = [ "anyhow", "hyper", "hyperlocal", + "kata-types", "tokio", ] diff --git a/src/tools/kata-ctl/Makefile b/src/tools/kata-ctl/Makefile index 23ae7ca1e..546f0783a 100644 --- a/src/tools/kata-ctl/Makefile +++ b/src/tools/kata-ctl/Makefile @@ -52,13 +52,13 @@ clean: vendor: cargo vendor -test: +test: $(GENERATED_CODE) @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo test --target $(TRIPLE) $(if $(findstring release,$(BUILD_TYPE)),--release) $(EXTRA_RUSTFEATURES) -- --nocapture install: @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo install --locked --target $(TRIPLE) --path . --root $(INSTALL_PATH) -check: standard_rust_check +check: $(GENERATED_CODE) standard_rust_check .PHONY: \ build \ diff --git a/src/tools/kata-ctl/src/check.rs b/src/tools/kata-ctl/src/check.rs index b579be176..0d5e01dcc 100644 --- a/src/tools/kata-ctl/src/check.rs +++ b/src/tools/kata-ctl/src/check.rs @@ -539,10 +539,10 @@ mod tests { }, // Success scenarios TestData { - module_name: "kvm", + module_name: "loop", param_name: "", kernel_module: &KernelModule { - name: "kvm", + name: "loop", params: &[KernelParam { name: "nonexistantparam", value: KernelParamType::Simple("Y"), @@ -552,16 +552,16 @@ mod tests { result: Ok(()), }, TestData { - module_name: "kvm", - param_name: "kvmclock_periodic_sync", + module_name: "loop", + param_name: "hw_queue_depth", kernel_module: &KernelModule { - name: "kvm", + name: "loop", params: &[KernelParam { - name: "kvmclock_periodic_sync", - value: KernelParamType::Simple("Y"), + name: "hw_queue_depth", + value: KernelParamType::Simple("128"), }], }, - param_value: "Y", + param_value: "128", result: Ok(()), }, ]; diff --git a/tests/common.bash b/tests/common.bash index fc0ed0a7f..fc5eac5cb 100644 --- a/tests/common.bash +++ b/tests/common.bash @@ -413,3 +413,42 @@ function install_cri_tools() { sudo tar -xvf "${tarball_name}" -C /usr/local/bin rm -f "${tarball_name}" } + +# Convert architecture to the name used by golang +function arch_to_golang() { + local arch="$(uname -m)" + + case "${arch}" in + aarch64) echo "arm64";; + ppc64le) echo "${arch}";; + x86_64) echo "amd64";; + s390x) echo "s390x";; + *) die "unsupported architecture: ${arch}";; + esac +} + +# Convert architecture to the name used by rust +function arch_to_rust() { + local -r arch="$(uname -m)" + + case "${arch}" in + aarch64) echo "${arch}";; + ppc64le) echo "powerpc64le";; + x86_64) echo "${arch}";; + s390x) echo "${arch}";; + *) die "unsupported architecture: ${arch}";; + esac +} + +# Convert architecture to the name used by the Linux kernel build system +function arch_to_kernel() { + local -r arch="$(uname -m)" + + case "${arch}" in + aarch64) echo "arm64";; + ppc64le) echo "powerpc";; + x86_64) echo "${arch}";; + s390x) echo "s390x";; + *) die "unsupported architecture: ${arch}";; + esac +} diff --git a/tests/install_go.sh b/tests/install_go.sh index 3827bb7b3..dadaf6ca0 100755 --- a/tests/install_go.sh +++ b/tests/install_go.sh @@ -87,7 +87,7 @@ if command -v go; then fi fi -goarch=$("${repo_root_dir}/tests/kata-arch.sh" --golang) +goarch=$(arch_to_golang) info "Download go version ${go_version}" kernel_name=$(uname -s) diff --git a/tests/install_rust.sh b/tests/install_rust.sh new file mode 100755 index 000000000..abb93cac6 --- /dev/null +++ b/tests/install_rust.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Copyright (c) 2019 Ant Financial +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o nounset +set -o pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +script_name="$(basename "${BASH_SOURCE[0]}")" + +source "${script_dir}/common.bash" + +rustarch=$(arch_to_rust) + +version="${1:-""}" +if [ -z "${version}" ]; then + version=$(get_from_kata_deps "languages.rust.meta.newest-version") +fi + +echo "Install rust ${version}" + +if ! command -v rustup > /dev/null; then + curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain ${version} +fi + +export PATH="${PATH}:${HOME}/.cargo/bin" + +## Still try to install the target version of toolchain, +## in case that the rustup has been installed but +## with a different version toolchain. +## Even though the target version toolchain has been installed, +## this command will not take too long to run. +rustup toolchain install ${version} +rustup default ${version} +if [ "${rustarch}" == "powerpc64le" ] || [ "${rustarch}" == "s390x" ] ; then + rustup target add ${rustarch}-unknown-linux-gnu +else + rustup target add ${rustarch}-unknown-linux-musl + $([ "$(whoami)" != "root" ] && echo sudo) ln -sf /usr/bin/g++ /bin/musl-g++ +fi +rustup component add rustfmt +rustup component add clippy