From 75c974c8024d931e085ffd214be4717e046b6640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 13:26:40 +0200 Subject: [PATCH 01/18] ci: static-checks: Move kernel config check to its own job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It doesn't make sense to run this for all the bits of the matrix, neither it's demanding enough to require running this in one of our Azure sponsored runners. Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- .github/workflows/static-checks.yaml | 37 +++++++++++++++++----------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/.github/workflows/static-checks.yaml b/.github/workflows/static-checks.yaml index cb113bfb0..ff952623c 100644 --- a/.github/workflows/static-checks.yaml +++ b/.github/workflows/static-checks.yaml @@ -12,6 +12,28 @@ concurrency: name: Static checks jobs: + check-kernel-config-version: + runs-on: ubuntu-latest + steps: + - name: Checkout the code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Ensure the kernel config version has been updated + run: | + kernel_dir="tools/packaging/kernel/" + kernel_version_file="${kernel_dir}kata_config_version" + modified_files=$(git diff --name-only origin/$GITHUB_BASE_REF..HEAD) + if git diff --name-only origin/$GITHUB_BASE_REF..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then + echo "Kernel directory has changed, checking if $kernel_version_file has been updated" + if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then + echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1) + else + echo "Readme file changed, no need for kernel config version update." + fi + echo "Check passed" + fi + static-checks: runs-on: garm-ubuntu-2004 strategy: @@ -48,21 +70,6 @@ jobs: uses: actions/setup-go@v3 with: go-version: 1.19.3 - - name: Check kernel config version - run: | - cd "${{ github.workspace }}/src/github.com/${{ github.repository }}" - kernel_dir="tools/packaging/kernel/" - kernel_version_file="${kernel_dir}kata_config_version" - modified_files=$(git diff --name-only origin/main..HEAD) - if git diff --name-only origin/main..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then - echo "Kernel directory has changed, checking if $kernel_version_file has been updated" - if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then - echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1) - else - echo "Readme file changed, no need for kernel config version update." - fi - echo "Check passed" - fi - name: Set PATH if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} run: | From 11dff731b74d46c5135098073bc47c0b6a43dfbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 13:59:41 +0200 Subject: [PATCH 02/18] tests: Move functions from kata_arch script here MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can use this a lot as part of our CI, but right now I'm just moving those here with the intent to use later on in this series. Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- tests/common.bash | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/common.bash b/tests/common.bash index fc0ed0a7f..fc5eac5cb 100644 --- a/tests/common.bash +++ b/tests/common.bash @@ -413,3 +413,42 @@ function install_cri_tools() { sudo tar -xvf "${tarball_name}" -C /usr/local/bin rm -f "${tarball_name}" } + +# Convert architecture to the name used by golang +function arch_to_golang() { + local arch="$(uname -m)" + + case "${arch}" in + aarch64) echo "arm64";; + ppc64le) echo "${arch}";; + x86_64) echo "amd64";; + s390x) echo "s390x";; + *) die "unsupported architecture: ${arch}";; + esac +} + +# Convert architecture to the name used by rust +function arch_to_rust() { + local -r arch="$(uname -m)" + + case "${arch}" in + aarch64) echo "${arch}";; + ppc64le) echo "powerpc64le";; + x86_64) echo "${arch}";; + s390x) echo "${arch}";; + *) die "unsupported architecture: ${arch}";; + esac +} + +# Convert architecture to the name used by the Linux kernel build system +function arch_to_kernel() { + local -r arch="$(uname -m)" + + case "${arch}" in + aarch64) echo "arm64";; + ppc64le) echo "powerpc";; + x86_64) echo "${arch}";; + s390x) echo "s390x";; + *) die "unsupported architecture: ${arch}";; + esac +} From e64508c3089eab09cfa8907f822fd2acd0e0acb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 14:01:37 +0200 Subject: [PATCH 03/18] tests: install_go: Remove tests repo dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can rely on the functions that are now part of the common.bash. Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- tests/install_go.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/install_go.sh b/tests/install_go.sh index 3827bb7b3..dadaf6ca0 100755 --- a/tests/install_go.sh +++ b/tests/install_go.sh @@ -87,7 +87,7 @@ if command -v go; then fi fi -goarch=$("${repo_root_dir}/tests/kata-arch.sh" --golang) +goarch=$(arch_to_golang) info "Download go version ${go_version}" kernel_name=$(uname -s) From 6794d4c843e9b5bf91c1b1fdae88585cab0dd7c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 14:06:18 +0200 Subject: [PATCH 04/18] tests: Move install_rust.sh from the tests repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We'll use it as part of the refactoring we're doing in the static check tests. I can see a lot of other uses of this, but changing all of them to this one is out of the scope for this PR. Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- tests/install_rust.sh | 44 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100755 tests/install_rust.sh diff --git a/tests/install_rust.sh b/tests/install_rust.sh new file mode 100755 index 000000000..6fc62ef02 --- /dev/null +++ b/tests/install_rust.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright (c) 2019 Ant Financial +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o nounset +set -o pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +script_name="$(basename "${BASH_SOURCE[0]}")" + +source "${script_dir}/common.bash" + +rustarch=$(arch_to_rust) + +version="${1:-""}" +if [ -z "${version}" ]; then + version=$(get_from_kata_deps "languages.rust.meta.newest-version") +fi + +echo "Install rust ${version}" + +if ! command -v rustup > /dev/null; then + curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain ${version} +fi + +export PATH="${PATH}:${HOME}/.cargo/bin" + +## Still try to install the target version of toolchain, +## in case that the rustup has been installed but +## with a different version toolchain. +## Even though the target version toolchain has been installed, +## this command will not take too long to run. +rustup toolchain install ${version} +rustup default ${version} +if [ "${rustarch}" == "powerpc64le" ] || [ "${rustarch}" == "s390x" ] ; then + rustup target add ${rustarch}-unknown-linux-gnu +else + rustup target add ${rustarch}-unknown-linux-musl + $([ "$(whoami)" != "root" ] && echo sudo) ln -sf /usr/bin/g++ /bin/musl-g++ +fi +rustup component add rustfmt From e2c61a152c0bfa7fc3ea34addbe67538fe3642e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 14:23:07 +0200 Subject: [PATCH 05/18] ci: static-checks: Move vendor check to its own job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Similarly to the static-check jobs, those jobs can be run on the zero cost runners. Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- .github/workflows/static-checks.yaml | 62 +++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/.github/workflows/static-checks.yaml b/.github/workflows/static-checks.yaml index ff952623c..393869174 100644 --- a/.github/workflows/static-checks.yaml +++ b/.github/workflows/static-checks.yaml @@ -34,6 +34,67 @@ jobs: echo "Check passed" fi + check-vendor: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + component: + - agent + - dragonball + - runtime + - runtime-rs + - agent-ctl + - kata-ctl + - log-parser-rs + - runk + - trace-forwarder + include: + - component: agent + component-path: src/agent + - component: dragonball + component-path: src/dragonball + - component: runtime + component-path: src/runtime + - component: runtime-rs + component-path: src/runtime-rs + - component: agent-ctl + component-path: src/tools/agent-ctl + - component: kata-ctl + component-path: src/tools/kata-ctl + - component: log-parser-rs + component-path: src/tools/log-parser-rs + - component: runk + component-path: src/tools/runk + - component: trace-forwarder + component-path: src/tools/trace-forwarder + steps: + - name: Checkout the code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install yq + run: | + ./ci/install_yq.sh + env: + INSTALL_IN_GOPATH: false + - name: Install golang + if: ${{ matrix.component == 'runtime' }} + run: | + ./tests/install_go.sh -f -p + echo "/usr/local/go/bin" >> $GITHUB_PATH + - name: Install rust + if: ${{ matrix.component != 'runtime' }} + run: | + ./tests/install_rust.sh + echo "${HOME}/.cargo/bin" >> $GITHUB_PATH + - name: Check ${{ matrix.component }} vendored code + run: | + cd ${{ matrix.component-path }} + make vendor + env: + RUST_BACKTRACE: "1" + static-checks: runs-on: garm-ubuntu-2004 strategy: @@ -43,7 +104,6 @@ jobs: fail-fast: false matrix: cmd: - - "make vendor" - "make static-checks" - "make check" - "make test" From e1257758634c2d27ac8518f1cf0b23febcf74f04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 15:28:09 +0200 Subject: [PATCH 06/18] tests: install_rust: Also install clippy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit clippy is used as part our tests, so it's useful to have it installed while we're already installing rust. In case of developers, they also better be using it. :-) Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- tests/install_rust.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/install_rust.sh b/tests/install_rust.sh index 6fc62ef02..abb93cac6 100755 --- a/tests/install_rust.sh +++ b/tests/install_rust.sh @@ -42,3 +42,4 @@ else $([ "$(whoami)" != "root" ] && echo sudo) ln -sf /usr/bin/g++ /bin/musl-g++ fi rustup component add rustfmt +rustup component add clippy From ea19549a997fc53a56435a75c4f7dda5c3ee1613 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 15:35:41 +0200 Subject: [PATCH 07/18] kata-ctl: Ensure GENERATED_CODE is a dep of `make check` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise `make check` would fail with: ``` Error writing files: failed to resolve mod `version`: /home/runner/work/kata-containers/kata-containers/src/tools/kata-ctl/src/ops/version.rs does not exist make: *** [../../../utils.mk:176: standard_rust_check] Error 1 ``` Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- src/tools/kata-ctl/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/kata-ctl/Makefile b/src/tools/kata-ctl/Makefile index 23ae7ca1e..ab677525e 100644 --- a/src/tools/kata-ctl/Makefile +++ b/src/tools/kata-ctl/Makefile @@ -58,7 +58,7 @@ test: install: @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo install --locked --target $(TRIPLE) --path . --root $(INSTALL_PATH) -check: standard_rust_check +check: $(GENERATED_CODE) standard_rust_check .PHONY: \ build \ From 473ec8780675b7377655ef3a56b73552b9bbbcf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 15:36:51 +0200 Subject: [PATCH 08/18] kata-ctl: Add `kata-types` to the Cargo.lock file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit message covered everything. :-) Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- src/tools/kata-ctl/Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tools/kata-ctl/Cargo.lock b/src/tools/kata-ctl/Cargo.lock index eda34fb17..2ddbcdbb9 100644 --- a/src/tools/kata-ctl/Cargo.lock +++ b/src/tools/kata-ctl/Cargo.lock @@ -1946,6 +1946,7 @@ dependencies = [ "anyhow", "hyper", "hyperlocal", + "kata-types", "tokio", ] From bf888b9a5eb954561d91c190e6fd352f0eb18914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 15:21:35 +0200 Subject: [PATCH 09/18] ci: static-checks: Move "make check" to the new test matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We're moving it out of the previous "static-checks" confusing matrix, and adding it to the matrix that was currently being used for the `make vendor` checks. This will allow us to have one job per component, and with that we can easily run those in parallel and on the zero cost runners. Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- .github/workflows/static-checks.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/static-checks.yaml b/.github/workflows/static-checks.yaml index 393869174..b2de664ac 100644 --- a/.github/workflows/static-checks.yaml +++ b/.github/workflows/static-checks.yaml @@ -34,7 +34,7 @@ jobs: echo "Check passed" fi - check-vendor: + build-checks: runs-on: ubuntu-20.04 strategy: fail-fast: false @@ -49,6 +49,9 @@ jobs: - log-parser-rs - runk - trace-forwarder + command: + - "make vendor" + - "make check" include: - component: agent component-path: src/agent @@ -88,10 +91,10 @@ jobs: run: | ./tests/install_rust.sh echo "${HOME}/.cargo/bin" >> $GITHUB_PATH - - name: Check ${{ matrix.component }} vendored code + - name: Running `${{ matrix.command }}` for ${{ matrix.component }} run: | cd ${{ matrix.component-path }} - make vendor + ${{ matrix.command }} env: RUST_BACKTRACE: "1" @@ -105,7 +108,6 @@ jobs: matrix: cmd: - "make static-checks" - - "make check" - "make test" - "sudo -E PATH=\"$PATH\" make test" env: From 1d32410a832c929b595e230ebe3e531f3d7d41a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 16:08:34 +0200 Subject: [PATCH 10/18] ci: install_libseccomp: Do not depend on the tests repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It makes things way simpler, waaaaay simpler. Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- ci/install_libseccomp.sh | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/ci/install_libseccomp.sh b/ci/install_libseccomp.sh index 683d0f65b..5d53be733 100755 --- a/ci/install_libseccomp.sh +++ b/ci/install_libseccomp.sh @@ -7,12 +7,10 @@ set -o errexit -cidir=$(dirname "$0") -source "${cidir}/lib.sh" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +script_name="$(basename "${BASH_SOURCE[0]}")" -clone_tests_repo - -source "${tests_repo_dir}/.ci/lib.sh" +source "${script_dir}/../tests/common.bash" # The following variables if set on the environment will change the behavior # of gperf and libseccomp configure scripts, that may lead this script to @@ -25,11 +23,11 @@ workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)" # Variables for libseccomp libseccomp_version="${LIBSECCOMP_VERSION:-""}" if [ -z "${libseccomp_version}" ]; then - libseccomp_version=$(get_version "externals.libseccomp.version") + libseccomp_version=$(get_from_kata_deps "externals.libseccomp.version") fi libseccomp_url="${LIBSECCOMP_URL:-""}" if [ -z "${libseccomp_url}" ]; then - libseccomp_url=$(get_version "externals.libseccomp.url") + libseccomp_url=$(get_from_kata_deps "externals.libseccomp.url") fi libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz" libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}" @@ -38,11 +36,11 @@ cflags="-O2" # Variables for gperf gperf_version="${GPERF_VERSION:-""}" if [ -z "${gperf_version}" ]; then - gperf_version=$(get_version "externals.gperf.version") + gperf_version=$(get_from_kata_deps "externals.gperf.version") fi gperf_url="${GPERF_URL:-""}" if [ -z "${gperf_url}" ]; then - gperf_url=$(get_version "externals.gperf.url") + gperf_url=$(get_from_kata_deps "externals.gperf.url") fi gperf_tarball="gperf-${gperf_version}.tar.gz" gperf_tarball_url="${gperf_url}/${gperf_tarball}" From ec826f328f88a1ff665c37381ada069eec337e65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 17:08:34 +0200 Subject: [PATCH 11/18] agent: Ensure GENERATED_CODE is a dep of `make test` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise `make test` will fail with: ``` error[E0583]: file not found for module `version` ``` Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- src/agent/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/Makefile b/src/agent/Makefile index d058caf64..699b71ce1 100644 --- a/src/agent/Makefile +++ b/src/agent/Makefile @@ -148,7 +148,7 @@ vendor: #TARGET test: run cargo tests -test: +test: $(GENERATED_FILES) @cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture ##TARGET check: run test From 46daddc5005c95718e26a2841515ee40823031ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 17:25:55 +0200 Subject: [PATCH 12/18] kata-ctl: Ensure GENERATED_CODE is a dep of `make test` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise `make test` will simply fail with: ``` error[E0583]: file not found for module `version` ``` Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- src/tools/kata-ctl/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/kata-ctl/Makefile b/src/tools/kata-ctl/Makefile index ab677525e..546f0783a 100644 --- a/src/tools/kata-ctl/Makefile +++ b/src/tools/kata-ctl/Makefile @@ -52,7 +52,7 @@ clean: vendor: cargo vendor -test: +test: $(GENERATED_CODE) @RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo test --target $(TRIPLE) $(if $(findstring release,$(BUILD_TYPE)),--release) $(EXTRA_RUSTFEATURES) -- --nocapture install: From 2bc3a616aec17aa145aaa7fbb839a8c3bb1104ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Sat, 16 Sep 2023 10:06:59 +0200 Subject: [PATCH 13/18] kata-ctl: Use `loop` instead of `kvm` module in tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes it pssible to run the tests in the cost free runners, which are not KVM capable. Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- src/tools/kata-ctl/src/check.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/tools/kata-ctl/src/check.rs b/src/tools/kata-ctl/src/check.rs index b579be176..0d5e01dcc 100644 --- a/src/tools/kata-ctl/src/check.rs +++ b/src/tools/kata-ctl/src/check.rs @@ -539,10 +539,10 @@ mod tests { }, // Success scenarios TestData { - module_name: "kvm", + module_name: "loop", param_name: "", kernel_module: &KernelModule { - name: "kvm", + name: "loop", params: &[KernelParam { name: "nonexistantparam", value: KernelParamType::Simple("Y"), @@ -552,16 +552,16 @@ mod tests { result: Ok(()), }, TestData { - module_name: "kvm", - param_name: "kvmclock_periodic_sync", + module_name: "loop", + param_name: "hw_queue_depth", kernel_module: &KernelModule { - name: "kvm", + name: "loop", params: &[KernelParam { - name: "kvmclock_periodic_sync", - value: KernelParamType::Simple("Y"), + name: "hw_queue_depth", + value: KernelParamType::Simple("128"), }], }, - param_value: "Y", + param_value: "128", result: Ok(()), }, ]; From 08f2e5ae0bbdc2e6c08b2ec29f29404207b2c2fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 18:15:53 +0200 Subject: [PATCH 14/18] runtime-rs: Ensure static-checks-build is a dep of `make test` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise `make test` will simply fail with: ``` error[E0583]: file not found for module `config` ``` Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- src/runtime-rs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 8dd730ac8..abc96a553 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -49,7 +49,7 @@ else ##TARGET default: build code default: runtime show-header ##TARGET test: run cargo tests -test: +test: static-checks-build @cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture install: install-runtime install-configs endif From 4e963cedf4f8fbddd112fb6461b0d191e54c3407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 15:51:45 +0200 Subject: [PATCH 15/18] ci: static-checks: Move "make test" to the new test matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We're moving it out of the previous "static-checks" confusing matrix, and adding it to the matrix that was currently being used for the `make vendor` and `make check` checks. This will allow us to have one job per component, and with that we can easily run those in parallel and on the zero cost runners. Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- .github/workflows/static-checks.yaml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/static-checks.yaml b/.github/workflows/static-checks.yaml index b2de664ac..7be939a4b 100644 --- a/.github/workflows/static-checks.yaml +++ b/.github/workflows/static-checks.yaml @@ -52,6 +52,7 @@ jobs: command: - "make vendor" - "make check" + - "make test" include: - component: agent component-path: src/agent @@ -71,6 +72,11 @@ jobs: component-path: src/tools/runk - component: trace-forwarder component-path: src/tools/trace-forwarder + - install-libseccomp: no + - component: agent + install-libseccomp: yes + - component: runk + install-libseccomp: yes steps: - name: Checkout the code uses: actions/checkout@v4 @@ -91,6 +97,23 @@ jobs: run: | ./tests/install_rust.sh echo "${HOME}/.cargo/bin" >> $GITHUB_PATH + - name: Install musl-tools + if: ${{ matrix.component != 'runtime' }} + run: sudo apt-get -y install musl-tools + - name: Install libseccomp + if: ${{ matrix.command != 'make vendor' && matrix.command != 'make check' && matrix.install-libseccomp == 'yes' }} + run: | + libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX) + gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX) + ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}" + echo "Set environment variables for the libseccomp crate to link the libseccomp library statically" + echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV + echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV + - name: Setup XDG_RUNTIME_DIR for the `runtime` tests + if: ${{ matrix.command != 'make vendor' && matrix.command != 'make check' && matrix.component == 'runtime' }} + run: | + XDG_RUNTIME_DIR=$(mktemp -d /tmp/kata-tests-$USER.XXX | tee >(xargs chmod 0700)) + echo "XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR}" >> $GITHUB_ENV - name: Running `${{ matrix.command }}` for ${{ matrix.component }} run: | cd ${{ matrix.component-path }} @@ -108,7 +131,6 @@ jobs: matrix: cmd: - "make static-checks" - - "make test" - "sudo -E PATH=\"$PATH\" make test" env: RUST_BACKTRACE: "1" From 509c309ab22096cd7c340908cf6c71ab15d3a197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 17:59:49 +0200 Subject: [PATCH 16/18] ci: static-checks: Move "sudo make test" to the new test matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We're moving it out of the previous "static-checks" confusing matrix, and adding it to the matrix that was currently being used for the `make vendor` and `make check` checks. This will allow us to have one job per component, and with that we can easily run those in parallel and on the zero cost runners. Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- .github/workflows/static-checks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/static-checks.yaml b/.github/workflows/static-checks.yaml index 7be939a4b..796330129 100644 --- a/.github/workflows/static-checks.yaml +++ b/.github/workflows/static-checks.yaml @@ -53,6 +53,7 @@ jobs: - "make vendor" - "make check" - "make test" + - "sudo -E PATH=\"$PATH\" make test" include: - component: agent component-path: src/agent @@ -131,7 +132,6 @@ jobs: matrix: cmd: - "make static-checks" - - "sudo -E PATH=\"$PATH\" make test" env: RUST_BACKTRACE: "1" target_branch: ${{ github.base_ref }} From 2c5ca2eaf816799a7dad3d47d5d52abac63c76ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 15:00:05 +0200 Subject: [PATCH 17/18] ci: static-checks: Run tests depending on KVM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With this we're removing the dragonball static-checks CI, as the test is running here now. :-) Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- .../workflows/static-checks-dragonball.yaml | 41 ------------------- .github/workflows/static-checks.yaml | 37 +++++++++++++++++ 2 files changed, 37 insertions(+), 41 deletions(-) delete mode 100644 .github/workflows/static-checks-dragonball.yaml diff --git a/.github/workflows/static-checks-dragonball.yaml b/.github/workflows/static-checks-dragonball.yaml deleted file mode 100644 index 2c99210a1..000000000 --- a/.github/workflows/static-checks-dragonball.yaml +++ /dev/null @@ -1,41 +0,0 @@ -on: - pull_request: - types: - - opened - - edited - - reopened - - synchronize - paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -name: Static checks dragonball -jobs: - test-dragonball: - runs-on: garm-ubuntu-2004 - env: - RUST_BACKTRACE: "1" - steps: - - uses: actions/checkout@v3 - - name: Set env - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends build-essential haveged - - name: Install Rust - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - ./ci/install_rust.sh - echo PATH="$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV - - name: Run Unit Test - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - cd src/dragonball - cargo version - rustc --version - sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test diff --git a/.github/workflows/static-checks.yaml b/.github/workflows/static-checks.yaml index 796330129..a6e27c7e7 100644 --- a/.github/workflows/static-checks.yaml +++ b/.github/workflows/static-checks.yaml @@ -122,6 +122,43 @@ jobs: env: RUST_BACKTRACE: "1" + build-checks-depending-on-kvm: + runs-on: garm-ubuntu-2004-smaller + strategy: + fail-fast: false + matrix: + component: + - runtime-rs + include: + - component: runtime-rs + command: "sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test" + - component: runtime-rs + component-path: src/dragonball + steps: + - name: Checkout the code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system deps + run: | + sudo apt-get install -y build-essential musl-tools + - name: Install yq + run: | + sudo -E ./ci/install_yq.sh + env: + INSTALL_IN_GOPATH: false + - name: Install rust + run: | + export PATH="$PATH:/usr/local/bin" + ./tests/install_rust.sh + - name: Running `${{ matrix.command }}` for ${{ matrix.component }} + run: | + export PATH="$PATH:${HOME}/.cargo/bin" + cd ${{ matrix.component-path }} + ${{ matrix.command }} + env: + RUST_BACKTRACE: "1" + static-checks: runs-on: garm-ubuntu-2004 strategy: From 8b1e9b0c758b947107fe7d73fec2e263ac01f2af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 15 Sep 2023 18:48:53 +0200 Subject: [PATCH 18/18] ci: static-checks: Clean up static-checks job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that the static-checks job only takes care of running the static-checks, let's clean it up, remove all the unneeded steps, make sure that we're using the actions in their latest version, and have it running in a cost free runner. At some point I'd like to see those tests done in parallel, in the same way that I've organised the build-checks, but that's something for someone else, at some other time. Fixes: #7974 -- part 0 Signed-off-by: Fabiano Fidêncio --- .github/workflows/static-checks.yaml | 77 +++++++++------------------- 1 file changed, 24 insertions(+), 53 deletions(-) diff --git a/.github/workflows/static-checks.yaml b/.github/workflows/static-checks.yaml index a6e27c7e7..ad2d2b7a4 100644 --- a/.github/workflows/static-checks.yaml +++ b/.github/workflows/static-checks.yaml @@ -160,64 +160,35 @@ jobs: RUST_BACKTRACE: "1" static-checks: - runs-on: garm-ubuntu-2004 + runs-on: ubuntu-20.04 strategy: - # We can set this to true whenever we're 100% sure that - # the all the tests are not flaky, otherwise we'll fail - # all the tests due to a single flaky instance. fail-fast: false matrix: cmd: - "make static-checks" env: - RUST_BACKTRACE: "1" - target_branch: ${{ github.base_ref }} GOPATH: ${{ github.workspace }} steps: - - name: Free disk space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - path: ./src/github.com/${{ github.repository }} - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends build-essential haveged - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: 1.19.3 - - name: Set PATH - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - echo "${{ github.workspace }}/bin" >> $GITHUB_PATH - - name: Setup - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh - - name: Installing rust - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh - PATH=$PATH:"$HOME/.cargo/bin" - rustup target add x86_64-unknown-linux-musl - rustup component add rustfmt clippy - - name: Setup seccomp - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX) - gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX) - cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}" - echo "Set environment variables for the libseccomp crate to link the libseccomp library statically" - echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV - echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV - - name: Run check - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - export PATH=$PATH:"$HOME/.cargo/bin" - export XDG_RUNTIME_DIR=$(mktemp -d /tmp/kata-tests-$USER.XXX | tee >(xargs chmod 0700)) - cd ${GOPATH}/src/github.com/${{ github.repository }} && ${{ matrix.cmd }} + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: ./src/github.com/${{ github.repository }} + - name: Install yq + run: | + cd ${GOPATH}/src/github.com/${{ github.repository }} + ./ci/install_yq.sh + env: + INSTALL_IN_GOPATH: false + - name: Install golang + run: | + cd ${GOPATH}/src/github.com/${{ github.repository }} + ./tests/install_go.sh -f -p + echo "/usr/local/go/bin" >> $GITHUB_PATH + - name: Install system dependencies + run: | + sudo apt-get -y install moreutils + - name: Run check + run: | + export PATH=${PATH}:${GOPATH}/bin + cd ${GOPATH}/src/github.com/${{ github.repository }} && ${{ matrix.cmd }}