gha: aks: Extract run commands to a script

Github Actions reads and runs workflow files from the main branch,
rather than from the PR branch. This means that PRs that modify workflow
files aren't being tested with the updated workflows coming from the PR,
but rather with the old workflows from the main branch. AFAIK, this
behavior isn't avoidable for workflow files (but is for other scripts).

This makes it very hard to reliably test workflow changes before they're
actually merged into main and leads to issues that we have to hotifx
(see #6983, #6995).

This PR aims to mitigate that by extracting the commands used in
workflows to a separate script file. The way our CI is set up, those
script files are read from the PR branch and thus changes would be
reflected in the CI checks.

Fixes: #6971

Signed-off-by: Aurélien Bombo <abombo@microsoft.com>

.github/workflows/run-k8s-tests-on-aks.yaml

@@ -27,83 +27,44 @@ jobs:
           - host_os: cbl-mariner
             vmm: clh
     runs-on: ubuntu-latest
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      GH_PR_NUMBER: ${{ github.event.pull_request.number }}
+      KATA_HOST_OS: ${{ matrix.host_os }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
     steps:
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Download Azure CLI
-        run: |
-          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
-          # The aks-preview extension is required while the Mariner Kata host is in preview.
-          az extension add --name aks-preview
+        run: bash tests/integration/gha-run.sh install-az-cli
 
       - name: Log into the Azure account
-        run: |
-          az login \
-            --service-principal \
-            -u "${{ secrets.AZ_APPID }}" \
-            -p "${{ secrets.AZ_PASSWORD }}" \
-            --tenant "${{ secrets.AZ_TENANT_ID }}"
-
-      - name: Format cluster name
-        run: |
-          rev=$(git rev-parse --short=12 HEAD)
-          echo "cluster_name=${{ github.event.pull_request.number }}-$rev-${{ matrix.vmm }}-${{ matrix.host_os }}-amd64" >> $GITHUB_ENV
+        run: bash tests/integration/gha-run.sh login-azure
+        env:
+          AZ_APPID: ${{ secrets.AZ_APPID }}
+          AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
+          AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
 
       - name: Create AKS cluster
-        run: |
-          az aks create \
-            -g "kataCI" \
-            -n "${{ env.cluster_name }}" \
-            -s "Standard_D4s_v5" \
-            --node-count 1 \
-            --generate-ssh-keys \
-            $([ "${{ matrix.host_os == 'cbl-mariner' }}" = "true" ] && echo "--os-sku mariner --workload-runtime KataMshvVmIsolation")
+        run: bash tests/integration/gha-run.sh create-cluster
 
       - name: Install `bats`
-        run: |
-          sudo apt-get update
-          sudo apt-get -y install bats
+        run: bash tests/integration/gha-run.sh install-bats
 
       - name: Install `kubectl`
-        run: |
-          sudo az aks install-cli
+        run: bash tests/integration/gha-run.sh install-kubectl
 
       - name: Download credentials for the Kubernetes CLI to use them
-        run: |
-          az aks get-credentials -g "kataCI" -n "${{ env.cluster_name }}"
+        run: bash tests/integration/gha-run.sh get-cluster-credentials
 
       - name: Run tests
         timeout-minutes: 60
-        run: |
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-
-          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
-
-          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
-          # which may cause issues like not having the node properly labeled or the artefacts
-          # properly deployed when the tests actually start running.
-          sleep 240s
-
-          pushd tests/integration/kubernetes
-          bash setup.sh
-          bash run_kubernetes_tests.sh
-          popd
-        env:
-          KATA_HOST_OS: ${{ matrix.host_os }}
-          KATA_HYPERVISOR: ${{ matrix.vmm }}
+        run: bash tests/integration/gha-run.sh run-tests-aks
 
       - name: Delete AKS cluster
         if: always()
-        run: |
-          az aks delete \
-            -g "kataCI" \
-            -n "${{ env.cluster_name }}" \
-            --yes \
-            --no-wait
+        run: bash tests/integration/gha-run.sh delete-cluster

.github/workflows/run-k8s-tests-on-sev.yaml

@@ -21,6 +21,10 @@ jobs:
           - qemu-sev
     runs-on: sev
     env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
       KUBECONFIG: /home/kata/.kube/config
     steps:
       - uses: actions/checkout@v3
@@ -29,40 +33,8 @@ jobs:
 
       - name: Run tests
         timeout-minutes: 30
-        run: |
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-
-          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
-
-          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
-          # which may cause issues like not having the node properly labeled or the artefacts
-          # properly deployed when the tests actually start running.
-          sleep 60s
-
-          pushd tests/integration/kubernetes
-          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
-          bash run_kubernetes_tests.sh
-          popd
-        env:
-          KATA_HYPERVISOR: ${{ matrix.vmm }}
+        run: bash tests/integration/gha-run.sh run-tests-sev
 
       - name: Delete kata-deploy
         if: always()
-        run: |
-          kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
-
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-          kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          sleep 180s
-
-          kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+        run: bash tests/integration/gha-run.sh cleanup-sev

.github/workflows/run-k8s-tests-on-snp.yaml

@@ -21,6 +21,10 @@ jobs:
           - qemu-snp
     runs-on: sev-snp
     env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
       KUBECONFIG: /home/kata/.kube/config
     steps:
       - uses: actions/checkout@v3
@@ -29,40 +33,8 @@ jobs:
 
       - name: Run tests
         timeout-minutes: 30
-        run: |
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-
-          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
-
-          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
-          # which may cause issues like not having the node properly labeled or the artefacts
-          # properly deployed when the tests actually start running.
-          sleep 60s
-
-          pushd tests/integration/kubernetes
-          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
-          bash run_kubernetes_tests.sh
-          popd
-        env:
-          KATA_HYPERVISOR: ${{ matrix.vmm }}
+        run: bash tests/integration/gha-run.sh run-tests-snp
 
       - name: Delete kata-deploy
         if: always()
-        run: |
-          kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
-
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-          kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          sleep 180s
-
-          kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+        run: bash tests/integration/gha-run.sh cleanup-snp

.github/workflows/run-k8s-tests-on-tdx.yaml

@@ -21,6 +21,10 @@ jobs:
           - qemu-tdx
     runs-on: tdx
     env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
       KUBECONFIG: /etc/rancher/k3s/k3s.yaml
     steps:
       - uses: actions/checkout@v3
@@ -29,40 +33,8 @@ jobs:
 
       - name: Run tests
         timeout-minutes: 30
-        run: |
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
-          cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-
-          kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl apply -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s
-          kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
-          kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
-
-          # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
-          # which may cause issues like not having the node properly labeled or the artefacts
-          # properly deployed when the tests actually start running.
-          sleep 60s
-
-          pushd tests/integration/kubernetes
-          sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
-          bash run_kubernetes_tests.sh
-          popd
-        env:
-          KATA_HYPERVISOR: ${{ matrix.vmm }}
+        run: bash tests/integration/gha-run.sh run-tests-tdx
 
       - name: Delete kata-deploy
         if: always()
-        run: |
-          kubectl delete -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s
-          kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
-
-          sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
-          cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
-          kubectl apply -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s
-          sleep 180s
-
-          kubectl delete -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s
-          kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
-          kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
+        run: bash tests/integration/gha-run.sh cleanup-tdx

tests/integration/gha-run.sh

@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2023 Microsoft Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+integration_dir="$(dirname "$(readlink -f "$0")")"
+tools_dir="${integration_dir}/../../tools"
+
+function _print_cluster_name() {
+    short_sha="$(git rev-parse --short=12 HEAD)"
+    echo "${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64"
+}
+
+function install_azure_cli() {
+    curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+    # The aks-preview extension is required while the Mariner Kata host is in preview.
+    az extension add --name aks-preview
+}
+
+function login_azure() {
+    az login \
+        --service-principal \
+        -u "${AZ_APPID}" \
+        -p "${AZ_PASSWORD}" \
+        --tenant "${AZ_TENANT_ID}"
+}
+
+function create_cluster() {
+    az aks create \
+        -g "kataCI" \
+        -n "$(_print_cluster_name)" \
+        -s "Standard_D4s_v5" \
+        --node-count 1 \
+        --generate-ssh-keys \
+        $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku mariner --workload-runtime KataMshvVmIsolation")
+}
+
+function install_bats() {
+    sudo apt-get update
+    sudo apt-get -y install bats
+}
+
+function install_kubectl() {
+    sudo az aks install-cli
+}
+
+function get_cluster_credentials() {
+    az aks get-credentials \
+        -g "kataCI" \
+        -n "$(_print_cluster_name)"
+}
+
+function run_tests() {
+    platform="${1}"
+
+    sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+    cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+    cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image"
+
+    kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
+    if [ "${platform}" = "tdx" ]; then
+        kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s"
+    else
+        kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+    fi
+    kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
+    kubectl apply -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml"
+
+    # This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
+    # which may cause issues like not having the node properly labeled or the artefacts
+    # properly deployed when the tests actually start running.
+    if [ "${platform}" = "aks" ]; then
+        sleep 240s
+    else
+        sleep 60s
+    fi
+
+    pushd "${integration_dir}/kubernetes"
+    bash setup.sh
+    bash run_kubernetes_tests.sh
+    popd
+}
+
+function cleanup() {
+    platform="${1}"
+
+    if [ "${platform}" = "tdx" ]; then
+        deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s""
+        cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k3s""
+    else
+        deploy_spec="-f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml""
+        cleanup_spec="-f "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml""
+    fi
+
+    kubectl delete ${deploy_spec}
+    kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
+
+    sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_IMAGE}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
+    cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
+    cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_IMAGE}:${DOCKER_TAG}" || die "Failed to setup the tests image"
+    kubectl apply ${cleanup_spec}
+    sleep 180s
+
+    kubectl delete ${cleanup_spec}
+    kubectl delete -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
+    kubectl delete -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml"
+}
+
+function delete_cluster() {
+    az aks delete \
+        -g "kataCI" \
+        -n "$(_print_cluster_name)" \
+        --yes \
+        --no-wait
+}
+
+function main() {
+    action="${1:-}"
+
+    case "${action}" in
+        install-azure-cli) install_azure_cli ;;
+        login-azure) login_azure ;;
+        create-cluster) create_cluster ;;
+        install-bats) install_bats ;;
+        install-kubectl) install_kubectl ;;
+        get-cluster-credentials) get_cluster_credentials ;;
+        run-tests-aks) run_tests "aks" ;;
+        run-tests-sev) run_tests "sev" ;;
+        run-tests-snp) run_tests "snp" ;;
+        run-tests-tdx) run_tests "tdx" ;;
+        cleanup-sev) cleanup "sev" ;;
+        cleanup-snp) cleanup "snp" ;;
+        cleanup-tdx) cleanup "tdx" ;;
+        delete-cluster) delete_cluster ;;
+        *) >&2 echo "Invalid action"; exit 2 ;;
+    esac
+}
+
+main "$@"