From aab60309620097aba19ac2c8b70f28d5dee774f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bombo?= Date: Fri, 26 May 2023 12:46:03 -0700 Subject: [PATCH] gha: aks: Extract `run` commands to a script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Github Actions reads and runs workflow files from the main branch, rather than from the PR branch. This means that PRs that modify workflow files aren't being tested with the updated workflows coming from the PR, but rather with the old workflows from the main branch. AFAIK, this behavior isn't avoidable for workflow files (but is for other scripts). This makes it very hard to reliably test workflow changes before they're actually merged into main and leads to issues that we have to hotifx (see #6983, #6995). This PR aims to mitigate that by extracting the commands used in workflows to a separate script file. The way our CI is set up, those script files are read from the PR branch and thus changes would be reflected in the CI checks. Fixes: #6971 Signed-off-by: Aurélien Bombo --- .github/workflows/run-k8s-tests-on-aks.yaml | 77 +++-------- .github/workflows/run-k8s-tests-on-sev.yaml | 40 +----- .github/workflows/run-k8s-tests-on-snp.yaml | 40 +----- .github/workflows/run-k8s-tests-on-tdx.yaml | 40 +----- tests/integration/gha-run.sh | 144 ++++++++++++++++++++ 5 files changed, 181 insertions(+), 160 deletions(-) create mode 100755 tests/integration/gha-run.sh diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index 7972f09e7d..e9f7b55b16 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -27,83 +27,44 @@ jobs: - host_os: cbl-mariner vmm: clh runs-on: ubuntu-latest + env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + GH_PR_NUMBER: ${{ github.event.pull_request.number }} + KATA_HOST_OS: ${{ matrix.host_os }} + KATA_HYPERVISOR: ${{ matrix.vmm }} steps: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - name: Download Azure CLI - run: | - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - # The aks-preview extension is required while the Mariner Kata host is in preview. - az extension add --name aks-preview + run: bash tests/integration/gha-run.sh install-az-cli - name: Log into the Azure account - run: | - az login \ - --service-principal \ - -u "${{ secrets.AZ_APPID }}" \ - -p "${{ secrets.AZ_PASSWORD }}" \ - --tenant "${{ secrets.AZ_TENANT_ID }}" - - - name: Format cluster name - run: | - rev=$(git rev-parse --short=12 HEAD) - echo "cluster_name=${{ github.event.pull_request.number }}-$rev-${{ matrix.vmm }}-${{ matrix.host_os }}-amd64" >> $GITHUB_ENV + run: bash tests/integration/gha-run.sh login-azure + env: + AZ_APPID: ${{ secrets.AZ_APPID }} + AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }} + AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }} - name: Create AKS cluster - run: | - az aks create \ - -g "kataCI" \ - -n "${{ env.cluster_name }}" \ - -s "Standard_D4s_v5" \ - --node-count 1 \ - --generate-ssh-keys \ - $([ "${{ matrix.host_os == 'cbl-mariner' }}" = "true" ] && echo "--os-sku mariner --workload-runtime KataMshvVmIsolation") + run: bash tests/integration/gha-run.sh create-cluster - name: Install `bats` - run: | - sudo apt-get update - sudo apt-get -y install bats + run: bash tests/integration/gha-run.sh install-bats - name: Install `kubectl` - run: | - sudo az aks install-cli + run: bash tests/integration/gha-run.sh install-kubectl - name: Download credentials for the Kubernetes CLI to use them - run: | - az aks get-credentials -g "kataCI" -n "${{ env.cluster_name }}" + run: bash tests/integration/gha-run.sh get-cluster-credentials - name: Run tests timeout-minutes: 60 - run: | - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - - kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - sleep 240s - - pushd tests/integration/kubernetes - bash setup.sh - bash run_kubernetes_tests.sh - popd - env: - KATA_HOST_OS: ${{ matrix.host_os }} - KATA_HYPERVISOR: ${{ matrix.vmm }} + run: bash tests/integration/gha-run.sh run-tests-aks - name: Delete AKS cluster if: always() - run: | - az aks delete \ - -g "kataCI" \ - -n "${{ env.cluster_name }}" \ - --yes \ - --no-wait + run: bash tests/integration/gha-run.sh delete-cluster diff --git a/.github/workflows/run-k8s-tests-on-sev.yaml b/.github/workflows/run-k8s-tests-on-sev.yaml index 98a6db6107..52ab7f9558 100644 --- a/.github/workflows/run-k8s-tests-on-sev.yaml +++ b/.github/workflows/run-k8s-tests-on-sev.yaml @@ -21,6 +21,10 @@ jobs: - qemu-sev runs-on: sev env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + KATA_HYPERVISOR: ${{ matrix.vmm }} KUBECONFIG: /home/kata/.kube/config steps: - uses: actions/checkout@v3 @@ -29,40 +33,8 @@ jobs: - name: Run tests timeout-minutes: 30 - run: | - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - - kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - sleep 60s - - pushd tests/integration/kubernetes - sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml - bash run_kubernetes_tests.sh - popd - env: - KATA_HYPERVISOR: ${{ matrix.vmm }} + run: bash tests/integration/gha-run.sh run-tests-sev - name: Delete kata-deploy if: always() - run: | - kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod - - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - sleep 180s - - kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml + run: bash tests/integration/gha-run.sh cleanup-sev diff --git a/.github/workflows/run-k8s-tests-on-snp.yaml b/.github/workflows/run-k8s-tests-on-snp.yaml index 541695e0f4..535c6de6dd 100644 --- a/.github/workflows/run-k8s-tests-on-snp.yaml +++ b/.github/workflows/run-k8s-tests-on-snp.yaml @@ -21,6 +21,10 @@ jobs: - qemu-snp runs-on: sev-snp env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + KATA_HYPERVISOR: ${{ matrix.vmm }} KUBECONFIG: /home/kata/.kube/config steps: - uses: actions/checkout@v3 @@ -29,40 +33,8 @@ jobs: - name: Run tests timeout-minutes: 30 - run: | - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - - kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - sleep 60s - - pushd tests/integration/kubernetes - sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml - bash run_kubernetes_tests.sh - popd - env: - KATA_HYPERVISOR: ${{ matrix.vmm }} + run: bash tests/integration/gha-run.sh run-tests-snp - name: Delete kata-deploy if: always() - run: | - kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod - - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - sleep 180s - - kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml + run: bash tests/integration/gha-run.sh cleanup-snp diff --git a/.github/workflows/run-k8s-tests-on-tdx.yaml b/.github/workflows/run-k8s-tests-on-tdx.yaml index 6c5ba0dc19..886b1c0268 100644 --- a/.github/workflows/run-k8s-tests-on-tdx.yaml +++ b/.github/workflows/run-k8s-tests-on-tdx.yaml @@ -21,6 +21,10 @@ jobs: - qemu-tdx runs-on: tdx env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + KATA_HYPERVISOR: ${{ matrix.vmm }} KUBECONFIG: /etc/rancher/k3s/k3s.yaml steps: - uses: actions/checkout@v3 @@ -29,40 +33,8 @@ jobs: - name: Run tests timeout-minutes: 30 - run: | - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - - kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl apply -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s - kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - sleep 60s - - pushd tests/integration/kubernetes - sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml - bash run_kubernetes_tests.sh - popd - env: - KATA_HYPERVISOR: ${{ matrix.vmm }} + run: bash tests/integration/gha-run.sh run-tests-tdx - name: Delete kata-deploy if: always() - run: | - kubectl delete -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s - kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod - - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - kubectl apply -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s - sleep 180s - - kubectl delete -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s - kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml + run: bash tests/integration/gha-run.sh cleanup-tdx diff --git a/tests/integration/gha-run.sh b/tests/integration/gha-run.sh new file mode 100755 index 0000000000..6238141dda --- /dev/null +++ b/tests/integration/gha-run.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash + +# Copyright (c) 2023 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o nounset +set -o pipefail + +integration_dir="$(dirname "$(readlink -f "$0")")" +tools_dir="${integration_dir}/../../tools" + +function _print_cluster_name() { + short_sha="$(git rev-parse --short=12 HEAD)" + echo "${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64" +} + +function install_azure_cli() { + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + # The aks-preview extension is required while the Mariner Kata host is in preview. + az extension add --name aks-preview +} + +function login_azure() { + az login \ + --service-principal \ + -u "${AZ_APPID}" \ + -p "${AZ_PASSWORD}" \ + --tenant "${AZ_TENANT_ID}" +} + +function create_cluster() { + az aks create \ + -g "kataCI" \ + -n "$(_print_cluster_name)" \ + -s "Standard_D4s_v5" \ + --node-count 1 \ + --generate-ssh-keys \ + $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku mariner --workload-runtime KataMshvVmIsolation") +} + +function install_bats() { + sudo apt-get update + sudo apt-get -y install bats +} + +function install_kubectl() { + sudo az aks install-cli +} + +function get_cluster_credentials() { + az aks get-credentials \ + -g "kataCI" \ + -n "$(_print_cluster_name)" +} + +function run_tests() { + platform="${1}" + + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image" + + kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" + if [ "${platform}" = "tdx" ]; then + kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s" + else + kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + fi + kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod + kubectl apply -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml" + + # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, + # which may cause issues like not having the node properly labeled or the artefacts + # properly deployed when the tests actually start running. + if [ "${platform}" = "aks" ]; then + sleep 240s + else + sleep 60s + fi + + pushd "${integration_dir}/kubernetes" + bash setup.sh + bash run_kubernetes_tests.sh + popd +} + +function cleanup() { + platform="${1}" + + if [ "${platform}" = "tdx" ]; then + deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s"" + cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k3s"" + else + deploy_spec="-f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"" + cleanup_spec="-f "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"" + fi + + kubectl delete ${deploy_spec} + kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod + + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_IMAGE}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" + cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" + cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_IMAGE}:${DOCKER_TAG}" || die "Failed to setup the tests image" + kubectl apply ${cleanup_spec} + sleep 180s + + kubectl delete ${cleanup_spec} + kubectl delete -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" + kubectl delete -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml" +} + +function delete_cluster() { + az aks delete \ + -g "kataCI" \ + -n "$(_print_cluster_name)" \ + --yes \ + --no-wait +} + +function main() { + action="${1:-}" + + case "${action}" in + install-azure-cli) install_azure_cli ;; + login-azure) login_azure ;; + create-cluster) create_cluster ;; + install-bats) install_bats ;; + install-kubectl) install_kubectl ;; + get-cluster-credentials) get_cluster_credentials ;; + run-tests-aks) run_tests "aks" ;; + run-tests-sev) run_tests "sev" ;; + run-tests-snp) run_tests "snp" ;; + run-tests-tdx) run_tests "tdx" ;; + cleanup-sev) cleanup "sev" ;; + cleanup-snp) cleanup "snp" ;; + cleanup-tdx) cleanup "tdx" ;; + delete-cluster) delete_cluster ;; + *) >&2 echo "Invalid action"; exit 2 ;; + esac +} + +main "$@"