From 40d514aa2c73ddca2ff0e3bb5615d62fd77e5ada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 4 Nov 2022 12:34:08 +0100 Subject: [PATCH] github: Parallelise static checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although introducing an awful amount of code duplication, let's parallelise the static checks in order to reduce its time and the space used in the VMs running those. While I understand there may be ways to make the whole setup less repetitive and error prone, I'm taking the approach of: * Make it work * Make it right * Make it fast So, it's clear that I'm only attempting to make it work, and I'd appreciate community help in order to improve the situation here. But, for now, this is a stopgap solution. JFYI, the time needed for run the tests on the `main` branch went down from ~110 minutes to ~60 minutes. Plus, we're not running those on a single VM anymore, which decreases the change to hit the space limit. Reference: https://github.com/kata-containers/kata-containers/actions/runs/3393468605/jobs/5640842041 Ideally, each one of the following tests should be also split into smaller tests, each test for one component, for instance. * static-checks * compiler-checks * unit-tests * unit-tests-as-root Fixes: #5585 Signed-off-by: Fabiano Fidêncio --- .github/workflows/static-checks.yaml | 258 ++++++++++++++++++++++++++- 1 file changed, 250 insertions(+), 8 deletions(-) diff --git a/.github/workflows/static-checks.yaml b/.github/workflows/static-checks.yaml index 0471a099bb..d0159adf04 100644 --- a/.github/workflows/static-checks.yaml +++ b/.github/workflows/static-checks.yaml @@ -8,7 +8,7 @@ on: name: Static checks jobs: - test: + check-vendored-code: strategy: matrix: go-version: [1.16.x, 1.17.x] @@ -24,7 +24,7 @@ jobs: steps: - name: Install Go if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: ${{ matrix.go-version }} env: @@ -43,7 +43,63 @@ jobs: echo "${{ github.workspace }}/bin" >> $GITHUB_PATH - name: Checkout code if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - uses: actions/checkout@v2 + uses: actions/checkout@v3 + with: + fetch-depth: 0 + path: ./src/github.com/${{ github.repository }} + - name: Setup travis references + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}" + target_branch=${TRAVIS_BRANCH} + - name: Setup + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh + env: + GOPATH: ${{ runner.workspace }}/kata-containers + # Check whether the vendored code is up-to-date & working as the first thing + - name: Check vendored code + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + cd ${GOPATH}/src/github.com/${{ github.repository }} && make vendor + + static-checks: + strategy: + matrix: + go-version: [1.16.x, 1.17.x] + os: [ubuntu-20.04] + runs-on: ${{ matrix.os }} + env: + TRAVIS: "true" + TRAVIS_BRANCH: ${{ github.base_ref }} + TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }} + TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }} + RUST_BACKTRACE: "1" + target_branch: ${{ github.base_ref }} + steps: + - name: Install Go + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + env: + GOPATH: ${{ runner.workspace }}/kata-containers + - name: Setup GOPATH + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}" + echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}" + echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}" + echo "TRAVIS: ${TRAVIS}" + - name: Set env + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV + echo "${{ github.workspace }}/bin" >> $GITHUB_PATH + - name: Checkout code + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + uses: actions/checkout@v3 with: fetch-depth: 0 path: ./src/github.com/${{ github.repository }} @@ -73,23 +129,209 @@ jobs: echo "Set environment variables for the libseccomp crate to link the libseccomp library statically" echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV - # Check whether the vendored code is up-to-date & working as the first thing - - name: Check vendored code - if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} - run: | - cd ${GOPATH}/src/github.com/${{ github.repository }} && make vendor - name: Static Checks if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} run: | cd ${GOPATH}/src/github.com/${{ github.repository }} && make static-checks + + + compiler-checks: + strategy: + matrix: + go-version: [1.16.x, 1.17.x] + os: [ubuntu-20.04] + runs-on: ${{ matrix.os }} + env: + TRAVIS: "true" + TRAVIS_BRANCH: ${{ github.base_ref }} + TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }} + TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }} + RUST_BACKTRACE: "1" + target_branch: ${{ github.base_ref }} + steps: + - name: Install Go + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + env: + GOPATH: ${{ runner.workspace }}/kata-containers + - name: Setup GOPATH + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}" + echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}" + echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}" + echo "TRAVIS: ${TRAVIS}" + - name: Set env + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV + echo "${{ github.workspace }}/bin" >> $GITHUB_PATH + - name: Checkout code + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + uses: actions/checkout@v3 + with: + fetch-depth: 0 + path: ./src/github.com/${{ github.repository }} + - name: Setup travis references + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}" + target_branch=${TRAVIS_BRANCH} + - name: Setup + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh + env: + GOPATH: ${{ runner.workspace }}/kata-containers + - name: Installing rust + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh + PATH=$PATH:"$HOME/.cargo/bin" + rustup target add x86_64-unknown-linux-musl + rustup component add rustfmt clippy + - name: Setup seccomp + run: | + libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX) + gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX) + cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}" + echo "Set environment variables for the libseccomp crate to link the libseccomp library statically" + echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV + echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV - name: Run Compiler Checks if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} run: | cd ${GOPATH}/src/github.com/${{ github.repository }} && make check + + unit-tests: + runs-on: ubuntu-20.04 + env: + TRAVIS: "true" + TRAVIS_BRANCH: ${{ github.base_ref }} + TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }} + TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }} + RUST_BACKTRACE: "1" + target_branch: ${{ github.base_ref }} + steps: + - name: Install Go + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + uses: actions/setup-go@v3 + with: + go-version: 1.17.x + env: + GOPATH: ${{ runner.workspace }}/kata-containers + - name: Setup GOPATH + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}" + echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}" + echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}" + echo "TRAVIS: ${TRAVIS}" + - name: Set env + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV + echo "${{ github.workspace }}/bin" >> $GITHUB_PATH + - name: Checkout code + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + uses: actions/checkout@v3 + with: + fetch-depth: 0 + path: ./src/github.com/${{ github.repository }} + - name: Setup travis references + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}" + target_branch=${TRAVIS_BRANCH} + - name: Setup + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh + env: + GOPATH: ${{ runner.workspace }}/kata-containers + - name: Installing rust + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh + PATH=$PATH:"$HOME/.cargo/bin" + rustup target add x86_64-unknown-linux-musl + rustup component add rustfmt clippy + - name: Setup seccomp + run: | + libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX) + gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX) + cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}" + echo "Set environment variables for the libseccomp crate to link the libseccomp library statically" + echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV + echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV - name: Run Unit Tests if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} run: | cd ${GOPATH}/src/github.com/${{ github.repository }} && make test + + unit-tests-as-root: + runs-on: ubuntu-20.04 + env: + TRAVIS: "true" + TRAVIS_BRANCH: ${{ github.base_ref }} + TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }} + TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }} + RUST_BACKTRACE: "1" + target_branch: ${{ github.base_ref }} + steps: + - name: Install Go + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + uses: actions/setup-go@v3 + with: + go-version: 1.17.x + env: + GOPATH: ${{ runner.workspace }}/kata-containers + - name: Setup GOPATH + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}" + echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}" + echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}" + echo "TRAVIS: ${TRAVIS}" + - name: Set env + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV + echo "${{ github.workspace }}/bin" >> $GITHUB_PATH + - name: Checkout code + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + uses: actions/checkout@v3 + with: + fetch-depth: 0 + path: ./src/github.com/${{ github.repository }} + - name: Setup travis references + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}" + target_branch=${TRAVIS_BRANCH} + - name: Setup + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh + env: + GOPATH: ${{ runner.workspace }}/kata-containers + - name: Installing rust + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: | + cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh + PATH=$PATH:"$HOME/.cargo/bin" + rustup target add x86_64-unknown-linux-musl + rustup component add rustfmt clippy + - name: Setup seccomp + run: | + libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX) + gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX) + cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}" + echo "Set environment variables for the libseccomp crate to link the libseccomp library statically" + echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV + echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV - name: Run Unit Tests As Root User if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} run: |