diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index 7972f09e7d..e9f7b55b16 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -27,83 +27,44 @@ jobs: - host_os: cbl-mariner vmm: clh runs-on: ubuntu-latest + env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + GH_PR_NUMBER: ${{ github.event.pull_request.number }} + KATA_HOST_OS: ${{ matrix.host_os }} + KATA_HYPERVISOR: ${{ matrix.vmm }} steps: - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - name: Download Azure CLI - run: | - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - # The aks-preview extension is required while the Mariner Kata host is in preview. - az extension add --name aks-preview + run: bash tests/integration/gha-run.sh install-az-cli - name: Log into the Azure account - run: | - az login \ - --service-principal \ - -u "${{ secrets.AZ_APPID }}" \ - -p "${{ secrets.AZ_PASSWORD }}" \ - --tenant "${{ secrets.AZ_TENANT_ID }}" - - - name: Format cluster name - run: | - rev=$(git rev-parse --short=12 HEAD) - echo "cluster_name=${{ github.event.pull_request.number }}-$rev-${{ matrix.vmm }}-${{ matrix.host_os }}-amd64" >> $GITHUB_ENV + run: bash tests/integration/gha-run.sh login-azure + env: + AZ_APPID: ${{ secrets.AZ_APPID }} + AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }} + AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }} - name: Create AKS cluster - run: | - az aks create \ - -g "kataCI" \ - -n "${{ env.cluster_name }}" \ - -s "Standard_D4s_v5" \ - --node-count 1 \ - --generate-ssh-keys \ - $([ "${{ matrix.host_os == 'cbl-mariner' }}" = "true" ] && echo "--os-sku mariner --workload-runtime KataMshvVmIsolation") + run: bash tests/integration/gha-run.sh create-cluster - name: Install `bats` - run: | - sudo apt-get update - sudo apt-get -y install bats + run: bash tests/integration/gha-run.sh install-bats - name: Install `kubectl` - run: | - sudo az aks install-cli + run: bash tests/integration/gha-run.sh install-kubectl - name: Download credentials for the Kubernetes CLI to use them - run: | - az aks get-credentials -g "kataCI" -n "${{ env.cluster_name }}" + run: bash tests/integration/gha-run.sh get-cluster-credentials - name: Run tests timeout-minutes: 60 - run: | - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - - kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - sleep 240s - - pushd tests/integration/kubernetes - bash setup.sh - bash run_kubernetes_tests.sh - popd - env: - KATA_HOST_OS: ${{ matrix.host_os }} - KATA_HYPERVISOR: ${{ matrix.vmm }} + run: bash tests/integration/gha-run.sh run-tests-aks - name: Delete AKS cluster if: always() - run: | - az aks delete \ - -g "kataCI" \ - -n "${{ env.cluster_name }}" \ - --yes \ - --no-wait + run: bash tests/integration/gha-run.sh delete-cluster diff --git a/.github/workflows/run-k8s-tests-on-sev.yaml b/.github/workflows/run-k8s-tests-on-sev.yaml index 98a6db6107..52ab7f9558 100644 --- a/.github/workflows/run-k8s-tests-on-sev.yaml +++ b/.github/workflows/run-k8s-tests-on-sev.yaml @@ -21,6 +21,10 @@ jobs: - qemu-sev runs-on: sev env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + KATA_HYPERVISOR: ${{ matrix.vmm }} KUBECONFIG: /home/kata/.kube/config steps: - uses: actions/checkout@v3 @@ -29,40 +33,8 @@ jobs: - name: Run tests timeout-minutes: 30 - run: | - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - - kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - sleep 60s - - pushd tests/integration/kubernetes - sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml - bash run_kubernetes_tests.sh - popd - env: - KATA_HYPERVISOR: ${{ matrix.vmm }} + run: bash tests/integration/gha-run.sh run-tests-sev - name: Delete kata-deploy if: always() - run: | - kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod - - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - sleep 180s - - kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml + run: bash tests/integration/gha-run.sh cleanup-sev diff --git a/.github/workflows/run-k8s-tests-on-snp.yaml b/.github/workflows/run-k8s-tests-on-snp.yaml index 541695e0f4..535c6de6dd 100644 --- a/.github/workflows/run-k8s-tests-on-snp.yaml +++ b/.github/workflows/run-k8s-tests-on-snp.yaml @@ -21,6 +21,10 @@ jobs: - qemu-snp runs-on: sev-snp env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + KATA_HYPERVISOR: ${{ matrix.vmm }} KUBECONFIG: /home/kata/.kube/config steps: - uses: actions/checkout@v3 @@ -29,40 +33,8 @@ jobs: - name: Run tests timeout-minutes: 30 - run: | - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - - kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - sleep 60s - - pushd tests/integration/kubernetes - sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml - bash run_kubernetes_tests.sh - popd - env: - KATA_HYPERVISOR: ${{ matrix.vmm }} + run: bash tests/integration/gha-run.sh run-tests-snp - name: Delete kata-deploy if: always() - run: | - kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod - - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - sleep 180s - - kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml + run: bash tests/integration/gha-run.sh cleanup-snp diff --git a/.github/workflows/run-k8s-tests-on-tdx.yaml b/.github/workflows/run-k8s-tests-on-tdx.yaml index 6c5ba0dc19..886b1c0268 100644 --- a/.github/workflows/run-k8s-tests-on-tdx.yaml +++ b/.github/workflows/run-k8s-tests-on-tdx.yaml @@ -21,6 +21,10 @@ jobs: - qemu-tdx runs-on: tdx env: + DOCKER_REGISTRY: ${{ inputs.registry }} + DOCKER_REPO: ${{ inputs.repo }} + DOCKER_TAG: ${{ inputs.tag }} + KATA_HYPERVISOR: ${{ matrix.vmm }} KUBECONFIG: /etc/rancher/k3s/k3s.yaml steps: - uses: actions/checkout@v3 @@ -29,40 +33,8 @@ jobs: - name: Run tests timeout-minutes: 30 - run: | - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml - cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - - kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl apply -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s - kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod - kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - sleep 60s - - pushd tests/integration/kubernetes - sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml - bash run_kubernetes_tests.sh - popd - env: - KATA_HYPERVISOR: ${{ matrix.vmm }} + run: bash tests/integration/gha-run.sh run-tests-tdx - name: Delete kata-deploy if: always() - run: | - kubectl delete -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s - kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod - - sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml - cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image" - kubectl apply -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s - sleep 180s - - kubectl delete -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s - kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml - kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml + run: bash tests/integration/gha-run.sh cleanup-tdx diff --git a/tests/integration/gha-run.sh b/tests/integration/gha-run.sh new file mode 100755 index 0000000000..6238141dda --- /dev/null +++ b/tests/integration/gha-run.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash + +# Copyright (c) 2023 Microsoft Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o nounset +set -o pipefail + +integration_dir="$(dirname "$(readlink -f "$0")")" +tools_dir="${integration_dir}/../../tools" + +function _print_cluster_name() { + short_sha="$(git rev-parse --short=12 HEAD)" + echo "${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64" +} + +function install_azure_cli() { + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + # The aks-preview extension is required while the Mariner Kata host is in preview. + az extension add --name aks-preview +} + +function login_azure() { + az login \ + --service-principal \ + -u "${AZ_APPID}" \ + -p "${AZ_PASSWORD}" \ + --tenant "${AZ_TENANT_ID}" +} + +function create_cluster() { + az aks create \ + -g "kataCI" \ + -n "$(_print_cluster_name)" \ + -s "Standard_D4s_v5" \ + --node-count 1 \ + --generate-ssh-keys \ + $([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku mariner --workload-runtime KataMshvVmIsolation") +} + +function install_bats() { + sudo apt-get update + sudo apt-get -y install bats +} + +function install_kubectl() { + sudo az aks install-cli +} + +function get_cluster_credentials() { + az aks get-credentials \ + -g "kataCI" \ + -n "$(_print_cluster_name)" +} + +function run_tests() { + platform="${1}" + + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image" + + kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" + if [ "${platform}" = "tdx" ]; then + kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s" + else + kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" + fi + kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod + kubectl apply -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml" + + # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, + # which may cause issues like not having the node properly labeled or the artefacts + # properly deployed when the tests actually start running. + if [ "${platform}" = "aks" ]; then + sleep 240s + else + sleep 60s + fi + + pushd "${integration_dir}/kubernetes" + bash setup.sh + bash run_kubernetes_tests.sh + popd +} + +function cleanup() { + platform="${1}" + + if [ "${platform}" = "tdx" ]; then + deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s"" + cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k3s"" + else + deploy_spec="-f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"" + cleanup_spec="-f "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"" + fi + + kubectl delete ${deploy_spec} + kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod + + sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_IMAGE}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" + cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" + cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_IMAGE}:${DOCKER_TAG}" || die "Failed to setup the tests image" + kubectl apply ${cleanup_spec} + sleep 180s + + kubectl delete ${cleanup_spec} + kubectl delete -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" + kubectl delete -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml" +} + +function delete_cluster() { + az aks delete \ + -g "kataCI" \ + -n "$(_print_cluster_name)" \ + --yes \ + --no-wait +} + +function main() { + action="${1:-}" + + case "${action}" in + install-azure-cli) install_azure_cli ;; + login-azure) login_azure ;; + create-cluster) create_cluster ;; + install-bats) install_bats ;; + install-kubectl) install_kubectl ;; + get-cluster-credentials) get_cluster_credentials ;; + run-tests-aks) run_tests "aks" ;; + run-tests-sev) run_tests "sev" ;; + run-tests-snp) run_tests "snp" ;; + run-tests-tdx) run_tests "tdx" ;; + cleanup-sev) cleanup "sev" ;; + cleanup-snp) cleanup "snp" ;; + cleanup-tdx) cleanup "tdx" ;; + delete-cluster) delete_cluster ;; + *) >&2 echo "Invalid action"; exit 2 ;; + esac +} + +main "$@"