Merge pull request #7976 from fidencio/topic/ci-static-checks-rework-part-0

ci: Rework static checks

.github/workflows/static-checks-dragonball.yaml

@@ -1,41 +0,0 @@
-on:
-  pull_request:
-    types:
-      - opened
-      - edited
-      - reopened
-      - synchronize
-    paths-ignore: [ '**.md', '**.png', '**.jpg', '**.jpeg', '**.svg', '/docs/**' ]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-name: Static checks dragonball
-jobs:
-  test-dragonball:
-    runs-on: garm-ubuntu-2004
-    env:
-      RUST_BACKTRACE: "1"
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set env
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends build-essential haveged
-      - name: Install Rust
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          ./ci/install_rust.sh
-          echo PATH="$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV
-      - name: Run Unit Test
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          cd src/dragonball
-          cargo version
-          rustc --version
-          sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test

.github/workflows/static-checks.yaml

@@ -12,49 +12,19 @@ concurrency:
 
 name: Static checks
 jobs:
-  static-checks:
+  check-kernel-config-version:
-    runs-on: garm-ubuntu-2004
+    runs-on: ubuntu-latest
-    strategy:
-      # We can set this to true whenever we're 100% sure that
-      # the all the tests are not flaky, otherwise we'll fail
-      # all the tests due to a single flaky instance.
-      fail-fast: false
-      matrix:
-        cmd:
-          - "make vendor"
-          - "make static-checks"
-          - "make check"
-          - "make test"
-          - "sudo -E PATH=\"$PATH\" make test"
-    env:
-      RUST_BACKTRACE: "1"
-      target_branch: ${{ github.base_ref }}
-      GOPATH: ${{ github.workspace }}
     steps:
-    - name: Free disk space
+      - name: Checkout the code
-      run: |
+        uses: actions/checkout@v4
-        sudo rm -rf /usr/share/dotnet
-        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-    - name: Checkout code
-      uses: actions/checkout@v3
         with:
           fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
+      - name: Ensure the kernel config version has been updated
-    - name: Install dependencies
         run: |
-        sudo apt-get update
-        sudo apt-get install -y --no-install-recommends build-essential haveged
-    - name: Install Go
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.19.3
-    - name: Check kernel config version
-      run: |
-        cd "${{ github.workspace }}/src/github.com/${{ github.repository }}"
           kernel_dir="tools/packaging/kernel/"
           kernel_version_file="${kernel_dir}kata_config_version"
-        modified_files=$(git diff --name-only origin/main..HEAD)
+          modified_files=$(git diff --name-only origin/$GITHUB_BASE_REF..HEAD)
-        if git diff --name-only origin/main..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
+          if git diff --name-only origin/$GITHUB_BASE_REF..HEAD "${kernel_dir}" | grep "${kernel_dir}"; then
             echo "Kernel directory has changed, checking if $kernel_version_file has been updated"
             if echo "$modified_files" | grep -v "README.md" | grep "${kernel_dir}" >>"/dev/null"; then
               echo "$modified_files" | grep "$kernel_version_file" >>/dev/null || ( echo "Please bump version in $kernel_version_file" && exit 1)
@@ -63,33 +33,162 @@ jobs:
             fi
             echo "Check passed"
           fi
-    - name: Set PATH
+
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+  build-checks:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - agent
+          - dragonball
+          - runtime
+          - runtime-rs
+          - agent-ctl
+          - kata-ctl
+          - log-parser-rs
+          - runk
+          - trace-forwarder
+        command:
+          - "make vendor"
+          - "make check"
+          - "make test"
+          - "sudo -E PATH=\"$PATH\" make test"
+        include:
+          - component: agent
+            component-path: src/agent
+          - component: dragonball
+            component-path: src/dragonball
+          - component: runtime
+            component-path: src/runtime
+          - component: runtime-rs
+            component-path: src/runtime-rs
+          - component: agent-ctl
+            component-path: src/tools/agent-ctl
+          - component: kata-ctl
+            component-path: src/tools/kata-ctl
+          - component: log-parser-rs
+            component-path: src/tools/log-parser-rs
+          - component: runk
+            component-path: src/tools/runk
+          - component: trace-forwarder
+            component-path: src/tools/trace-forwarder
+          - install-libseccomp: no
+          - component: agent
+            install-libseccomp: yes
+          - component: runk
+            install-libseccomp: yes
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install yq
         run: |
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+          ./ci/install_yq.sh
-    - name: Setup
+        env:
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+          INSTALL_IN_GOPATH: false
+      - name: Install golang
+        if: ${{ matrix.component == 'runtime' }}
         run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
+          ./tests/install_go.sh -f -p
-    - name: Installing rust
+          echo "/usr/local/go/bin" >> $GITHUB_PATH
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      - name: Install rust
+        if: ${{ matrix.component != 'runtime' }}
         run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
+          ./tests/install_rust.sh
-        PATH=$PATH:"$HOME/.cargo/bin"
+          echo "${HOME}/.cargo/bin" >> $GITHUB_PATH
-        rustup target add x86_64-unknown-linux-musl
+      - name: Install musl-tools
-        rustup component add rustfmt clippy
+        if: ${{ matrix.component != 'runtime' }}
-    - name: Setup seccomp
+        run: sudo apt-get -y install musl-tools
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      - name: Install libseccomp
+        if: ${{ matrix.command != 'make vendor'  &&  matrix.command != 'make check' &&  matrix.install-libseccomp == 'yes' }}
         run: |
           libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
           gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
+          ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
           echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
           echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
           echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
-    - name: Run check
+      - name: Setup XDG_RUNTIME_DIR for the `runtime` tests
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        if: ${{ matrix.command != 'make vendor' && matrix.command != 'make check' && matrix.component == 'runtime' }}
         run: |
-        export PATH=$PATH:"$HOME/.cargo/bin"
+          XDG_RUNTIME_DIR=$(mktemp -d /tmp/kata-tests-$USER.XXX | tee >(xargs chmod 0700))
-        export XDG_RUNTIME_DIR=$(mktemp -d /tmp/kata-tests-$USER.XXX | tee >(xargs chmod 0700))
+          echo "XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR}" >> $GITHUB_ENV
+      - name: Running `${{ matrix.command }}` for ${{ matrix.component }}
+        run: |
+          cd ${{ matrix.component-path }}
+          ${{ matrix.command }}
+        env:
+          RUST_BACKTRACE: "1"
+
+  build-checks-depending-on-kvm:
+    runs-on: garm-ubuntu-2004-smaller
+    strategy:
+      fail-fast: false
+      matrix:
+        component:
+          - runtime-rs
+        include:
+          - component: runtime-rs
+            command: "sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test"
+          - component: runtime-rs
+            component-path: src/dragonball
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install system deps
+        run: |
+          sudo apt-get install -y build-essential musl-tools
+      - name: Install yq
+        run: |
+          sudo -E ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install rust
+        run: |
+          export PATH="$PATH:/usr/local/bin"
+          ./tests/install_rust.sh
+      - name: Running `${{ matrix.command }}` for ${{ matrix.component }}
+        run: |
+          export PATH="$PATH:${HOME}/.cargo/bin"
+          cd ${{ matrix.component-path }}
+          ${{ matrix.command }}
+        env:
+          RUST_BACKTRACE: "1"
+
+  static-checks:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        cmd:
+          - "make static-checks"
+    env:
+      GOPATH: ${{ github.workspace }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          path: ./src/github.com/${{ github.repository }}
+      - name: Install yq
+        run: |
+          cd ${GOPATH}/src/github.com/${{ github.repository }}
+          ./ci/install_yq.sh
+        env:
+          INSTALL_IN_GOPATH: false
+      - name: Install golang
+        run: |
+          cd ${GOPATH}/src/github.com/${{ github.repository }}
+          ./tests/install_go.sh -f -p
+          echo "/usr/local/go/bin" >> $GITHUB_PATH
+      - name: Install system dependencies
+        run: |
+          sudo apt-get -y install moreutils
+      - name: Run check
+        run: |
+          export PATH=${PATH}:${GOPATH}/bin
           cd ${GOPATH}/src/github.com/${{ github.repository }} && ${{ matrix.cmd }}

ci/install_libseccomp.sh

@@ -7,12 +7,10 @@
 
 set -o errexit
 
-cidir=$(dirname "$0")
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source "${cidir}/lib.sh"
+script_name="$(basename "${BASH_SOURCE[0]}")"
 
-clone_tests_repo
+source "${script_dir}/../tests/common.bash"
-
-source "${tests_repo_dir}/.ci/lib.sh"
 
 # The following variables if set on the environment will change the behavior
 # of gperf and libseccomp configure scripts, that may lead this script to
@@ -25,11 +23,11 @@ workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
 # Variables for libseccomp
 libseccomp_version="${LIBSECCOMP_VERSION:-""}"
 if [ -z "${libseccomp_version}" ]; then
-    libseccomp_version=$(get_version "externals.libseccomp.version")
+    libseccomp_version=$(get_from_kata_deps "externals.libseccomp.version")
 fi
 libseccomp_url="${LIBSECCOMP_URL:-""}"
 if [ -z "${libseccomp_url}" ]; then
-    libseccomp_url=$(get_version "externals.libseccomp.url")
+    libseccomp_url=$(get_from_kata_deps "externals.libseccomp.url")
 fi
 libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz"
 libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}"
@@ -38,11 +36,11 @@ cflags="-O2"
 # Variables for gperf
 gperf_version="${GPERF_VERSION:-""}"
 if [ -z "${gperf_version}" ]; then
-    gperf_version=$(get_version "externals.gperf.version")
+    gperf_version=$(get_from_kata_deps "externals.gperf.version")
 fi
 gperf_url="${GPERF_URL:-""}"
 if [ -z "${gperf_url}" ]; then
-    gperf_url=$(get_version "externals.gperf.url")
+    gperf_url=$(get_from_kata_deps "externals.gperf.url")
 fi
 gperf_tarball="gperf-${gperf_version}.tar.gz"
 gperf_tarball_url="${gperf_url}/${gperf_tarball}"

src/agent/Makefile

@@ -148,7 +148,7 @@ vendor:
 
 
 #TARGET test: run cargo tests
-test:
+test: $(GENERATED_FILES)
 	@cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
 
 ##TARGET check: run test

src/runtime-rs/Makefile

@@ -49,7 +49,7 @@ else
 ##TARGET default: build code
 default: runtime show-header
 ##TARGET test: run cargo tests
-test:
+test: static-checks-build
 	@cargo test --all --target $(TRIPLE) $(EXTRA_RUSTFEATURES) -- --nocapture
 install: install-runtime install-configs
 endif

src/tools/kata-ctl/Cargo.lock

@@ -1946,6 +1946,7 @@ dependencies = [
  "anyhow",
  "hyper",
  "hyperlocal",
+ "kata-types",
  "tokio",
 ]
 

src/tools/kata-ctl/Makefile

@@ -52,13 +52,13 @@ clean:
 vendor:
 	cargo vendor
 
-test:
+test: $(GENERATED_CODE)
 	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo test --target $(TRIPLE) $(if $(findstring release,$(BUILD_TYPE)),--release) $(EXTRA_RUSTFEATURES) -- --nocapture
 
 install:
 	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo install --locked --target $(TRIPLE) --path . --root $(INSTALL_PATH)
 
-check: standard_rust_check
+check: $(GENERATED_CODE) standard_rust_check
 
 .PHONY: \
 	build \

src/tools/kata-ctl/src/check.rs

@@ -539,10 +539,10 @@ mod tests {
             },
             // Success scenarios
             TestData {
-                module_name: "kvm",
+                module_name: "loop",
                 param_name: "",
                 kernel_module: &KernelModule {
-                    name: "kvm",
+                    name: "loop",
                     params: &[KernelParam {
                         name: "nonexistantparam",
                         value: KernelParamType::Simple("Y"),
@@ -552,16 +552,16 @@ mod tests {
                 result: Ok(()),
             },
             TestData {
-                module_name: "kvm",
+                module_name: "loop",
-                param_name: "kvmclock_periodic_sync",
+                param_name: "hw_queue_depth",
                 kernel_module: &KernelModule {
-                    name: "kvm",
+                    name: "loop",
                     params: &[KernelParam {
-                        name: "kvmclock_periodic_sync",
+                        name: "hw_queue_depth",
-                        value: KernelParamType::Simple("Y"),
+                        value: KernelParamType::Simple("128"),
                     }],
                 },
-                param_value: "Y",
+                param_value: "128",
                 result: Ok(()),
             },
         ];

tests/common.bash

@@ -413,3 +413,42 @@ function install_cri_tools() {
 	sudo tar -xvf "${tarball_name}" -C /usr/local/bin
 	rm -f "${tarball_name}"
 }
+
+# Convert architecture to the name used by golang
+function arch_to_golang() {
+	local arch="$(uname -m)"
+
+	case "${arch}" in
+		aarch64) echo "arm64";;
+		ppc64le) echo "${arch}";;
+		x86_64) echo "amd64";;
+		s390x) echo "s390x";;
+		*) die "unsupported architecture: ${arch}";;
+	esac
+}
+
+# Convert architecture to the name used by rust
+function arch_to_rust() {
+	local -r arch="$(uname -m)"
+
+	case "${arch}" in
+		aarch64) echo "${arch}";;
+		ppc64le) echo "powerpc64le";;
+		x86_64) echo "${arch}";;
+		s390x) echo "${arch}";;
+		*) die "unsupported architecture: ${arch}";;
+	esac
+}
+
+# Convert architecture to the name used by the Linux kernel build system
+function arch_to_kernel() {
+	local -r arch="$(uname -m)"
+
+	case "${arch}" in
+		aarch64) echo "arm64";;
+		ppc64le) echo "powerpc";;
+		x86_64) echo "${arch}";;
+		s390x) echo "s390x";;
+		*) die "unsupported architecture: ${arch}";;
+	esac
+}

tests/install_go.sh

@@ -87,7 +87,7 @@ if command -v go; then
 	fi
 fi
 
-goarch=$("${repo_root_dir}/tests/kata-arch.sh" --golang)
+goarch=$(arch_to_golang)
 
 info "Download go version ${go_version}"
 kernel_name=$(uname -s)

tests/install_rust.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# Copyright (c) 2019 Ant Financial
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+script_name="$(basename "${BASH_SOURCE[0]}")"
+
+source "${script_dir}/common.bash"
+
+rustarch=$(arch_to_rust)
+
+version="${1:-""}"
+if [ -z "${version}" ]; then
+	version=$(get_from_kata_deps "languages.rust.meta.newest-version")
+fi
+
+echo "Install rust ${version}"
+
+if ! command -v rustup > /dev/null; then
+	curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain ${version}
+fi
+
+export PATH="${PATH}:${HOME}/.cargo/bin"
+
+## Still try to install the target version of toolchain,
+## in case that the rustup has been installed but
+## with a different version toolchain.
+## Even though the target version toolchain has been installed,
+## this command will not take too long to run.
+rustup toolchain install ${version}
+rustup default ${version}
+if [ "${rustarch}" == "powerpc64le" ] || [ "${rustarch}" == "s390x" ] ; then
+	rustup target add ${rustarch}-unknown-linux-gnu
+else
+	rustup target add ${rustarch}-unknown-linux-musl
+	$([ "$(whoami)" != "root" ] && echo sudo) ln -sf /usr/bin/g++ /bin/musl-g++
+fi
+rustup component add rustfmt
+rustup component add clippy