Merge pull request #6972 from sprt/sprt/gha-run-script

gha: aks: Extract `run` commands to a script
This commit is contained in:
Fabiano Fidêncio
2023-06-06 14:54:03 +02:00
committed by GitHub
5 changed files with 181 additions and 160 deletions

View File

@@ -27,83 +27,44 @@ jobs:
- host_os: cbl-mariner
vmm: clh
runs-on: ubuntu-latest
env:
DOCKER_REGISTRY: ${{ inputs.registry }}
DOCKER_REPO: ${{ inputs.repo }}
DOCKER_TAG: ${{ inputs.tag }}
GH_PR_NUMBER: ${{ github.event.pull_request.number }}
KATA_HOST_OS: ${{ matrix.host_os }}
KATA_HYPERVISOR: ${{ matrix.vmm }}
steps:
- uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Download Azure CLI
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
# The aks-preview extension is required while the Mariner Kata host is in preview.
az extension add --name aks-preview
run: bash tests/integration/gha-run.sh install-az-cli
- name: Log into the Azure account
run: |
az login \
--service-principal \
-u "${{ secrets.AZ_APPID }}" \
-p "${{ secrets.AZ_PASSWORD }}" \
--tenant "${{ secrets.AZ_TENANT_ID }}"
- name: Format cluster name
run: |
rev=$(git rev-parse --short=12 HEAD)
echo "cluster_name=${{ github.event.pull_request.number }}-$rev-${{ matrix.vmm }}-${{ matrix.host_os }}-amd64" >> $GITHUB_ENV
run: bash tests/integration/gha-run.sh login-azure
env:
AZ_APPID: ${{ secrets.AZ_APPID }}
AZ_PASSWORD: ${{ secrets.AZ_PASSWORD }}
AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }}
- name: Create AKS cluster
run: |
az aks create \
-g "kataCI" \
-n "${{ env.cluster_name }}" \
-s "Standard_D4s_v5" \
--node-count 1 \
--generate-ssh-keys \
$([ "${{ matrix.host_os == 'cbl-mariner' }}" = "true" ] && echo "--os-sku mariner --workload-runtime KataMshvVmIsolation")
run: bash tests/integration/gha-run.sh create-cluster
- name: Install `bats`
run: |
sudo apt-get update
sudo apt-get -y install bats
run: bash tests/integration/gha-run.sh install-bats
- name: Install `kubectl`
run: |
sudo az aks install-cli
run: bash tests/integration/gha-run.sh install-kubectl
- name: Download credentials for the Kubernetes CLI to use them
run: |
az aks get-credentials -g "kataCI" -n "${{ env.cluster_name }}"
run: bash tests/integration/gha-run.sh get-cluster-credentials
- name: Run tests
timeout-minutes: 60
run: |
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
# This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
# which may cause issues like not having the node properly labeled or the artefacts
# properly deployed when the tests actually start running.
sleep 240s
pushd tests/integration/kubernetes
bash setup.sh
bash run_kubernetes_tests.sh
popd
env:
KATA_HOST_OS: ${{ matrix.host_os }}
KATA_HYPERVISOR: ${{ matrix.vmm }}
run: bash tests/integration/gha-run.sh run-tests-aks
- name: Delete AKS cluster
if: always()
run: |
az aks delete \
-g "kataCI" \
-n "${{ env.cluster_name }}" \
--yes \
--no-wait
run: bash tests/integration/gha-run.sh delete-cluster

View File

@@ -21,6 +21,10 @@ jobs:
- qemu-sev
runs-on: sev
env:
DOCKER_REGISTRY: ${{ inputs.registry }}
DOCKER_REPO: ${{ inputs.repo }}
DOCKER_TAG: ${{ inputs.tag }}
KATA_HYPERVISOR: ${{ matrix.vmm }}
KUBECONFIG: /home/kata/.kube/config
steps:
- uses: actions/checkout@v3
@@ -29,40 +33,8 @@ jobs:
- name: Run tests
timeout-minutes: 30
run: |
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
# This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
# which may cause issues like not having the node properly labeled or the artefacts
# properly deployed when the tests actually start running.
sleep 60s
pushd tests/integration/kubernetes
sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
bash run_kubernetes_tests.sh
popd
env:
KATA_HYPERVISOR: ${{ matrix.vmm }}
run: bash tests/integration/gha-run.sh run-tests-sev
- name: Delete kata-deploy
if: always()
run: |
kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
sleep 180s
kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
run: bash tests/integration/gha-run.sh cleanup-sev

View File

@@ -21,6 +21,10 @@ jobs:
- qemu-snp
runs-on: sev-snp
env:
DOCKER_REGISTRY: ${{ inputs.registry }}
DOCKER_REPO: ${{ inputs.repo }}
DOCKER_TAG: ${{ inputs.tag }}
KATA_HYPERVISOR: ${{ matrix.vmm }}
KUBECONFIG: /home/kata/.kube/config
steps:
- uses: actions/checkout@v3
@@ -29,40 +33,8 @@ jobs:
- name: Run tests
timeout-minutes: 30
run: |
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
kubectl apply -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
# This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
# which may cause issues like not having the node properly labeled or the artefacts
# properly deployed when the tests actually start running.
sleep 60s
pushd tests/integration/kubernetes
sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
bash run_kubernetes_tests.sh
popd
env:
KATA_HYPERVISOR: ${{ matrix.vmm }}
run: bash tests/integration/gha-run.sh run-tests-snp
- name: Delete kata-deploy
if: always()
run: |
kubectl delete -f tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
kubectl apply -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
sleep 180s
kubectl delete -f tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
run: bash tests/integration/gha-run.sh cleanup-snp

View File

@@ -21,6 +21,10 @@ jobs:
- qemu-tdx
runs-on: tdx
env:
DOCKER_REGISTRY: ${{ inputs.registry }}
DOCKER_REPO: ${{ inputs.repo }}
DOCKER_TAG: ${{ inputs.tag }}
KATA_HYPERVISOR: ${{ matrix.vmm }}
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
steps:
- uses: actions/checkout@v3
@@ -29,40 +33,8 @@ jobs:
- name: Run tests
timeout-minutes: 30
run: |
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml
cat tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
kubectl apply -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
kubectl apply -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s
kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
kubectl apply -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
# This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
# which may cause issues like not having the node properly labeled or the artefacts
# properly deployed when the tests actually start running.
sleep 60s
pushd tests/integration/kubernetes
sed -i -e 's|runtimeClassName: kata|runtimeClassName: kata-${{ matrix.vmm }}|' runtimeclass_workloads/*.yaml
bash run_kubernetes_tests.sh
popd
env:
KATA_HYPERVISOR: ${{ matrix.vmm }}
run: bash tests/integration/gha-run.sh run-tests-tdx
- name: Delete kata-deploy
if: always()
run: |
kubectl delete -k tools/packaging/kata-deploy/kata-deploy/overlays/k3s
kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}|g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml
cat tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml | grep "${{ inputs.registry }}/${{ inputs.repo }}:${{ inputs.tag }}" || die "Failed to setup the tests image"
kubectl apply -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s
sleep 180s
kubectl delete -k tools/packaging/kata-deploy/kata-cleanup/overlays/k3s
kubectl delete -f tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml
kubectl delete -f tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
run: bash tests/integration/gha-run.sh cleanup-tdx

144
tests/integration/gha-run.sh Executable file
View File

@@ -0,0 +1,144 @@
#!/usr/bin/env bash
# Copyright (c) 2023 Microsoft Corporation
#
# SPDX-License-Identifier: Apache-2.0
set -o errexit
set -o nounset
set -o pipefail
integration_dir="$(dirname "$(readlink -f "$0")")"
tools_dir="${integration_dir}/../../tools"
function _print_cluster_name() {
short_sha="$(git rev-parse --short=12 HEAD)"
echo "${GH_PR_NUMBER}-${short_sha}-${KATA_HYPERVISOR}-${KATA_HOST_OS}-amd64"
}
function install_azure_cli() {
curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
# The aks-preview extension is required while the Mariner Kata host is in preview.
az extension add --name aks-preview
}
function login_azure() {
az login \
--service-principal \
-u "${AZ_APPID}" \
-p "${AZ_PASSWORD}" \
--tenant "${AZ_TENANT_ID}"
}
function create_cluster() {
az aks create \
-g "kataCI" \
-n "$(_print_cluster_name)" \
-s "Standard_D4s_v5" \
--node-count 1 \
--generate-ssh-keys \
$([ "${KATA_HOST_OS}" = "cbl-mariner" ] && echo "--os-sku mariner --workload-runtime KataMshvVmIsolation")
}
function install_bats() {
sudo apt-get update
sudo apt-get -y install bats
}
function install_kubectl() {
sudo az aks install-cli
}
function get_cluster_credentials() {
az aks get-credentials \
-g "kataCI" \
-n "$(_print_cluster_name)"
}
function run_tests() {
platform="${1}"
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" || die "Failed to setup the tests image"
kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
if [ "${platform}" = "tdx" ]; then
kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s"
else
kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
fi
kubectl -n kube-system wait --timeout=10m --for=condition=Ready -l name=kata-deploy pod
kubectl apply -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml"
# This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
# which may cause issues like not having the node properly labeled or the artefacts
# properly deployed when the tests actually start running.
if [ "${platform}" = "aks" ]; then
sleep 240s
else
sleep 60s
fi
pushd "${integration_dir}/kubernetes"
bash setup.sh
bash run_kubernetes_tests.sh
popd
}
function cleanup() {
platform="${1}"
if [ "${platform}" = "tdx" ]; then
deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s""
cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k3s""
else
deploy_spec="-f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml""
cleanup_spec="-f "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml""
fi
kubectl delete ${deploy_spec}
kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_IMAGE}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
cat "${tools_dir}/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" | grep "${DOCKER_REGISTRY}/${DOCKER_IMAGE}:${DOCKER_TAG}" || die "Failed to setup the tests image"
kubectl apply ${cleanup_spec}
sleep 180s
kubectl delete ${cleanup_spec}
kubectl delete -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
kubectl delete -f "${tools_dir}/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml"
}
function delete_cluster() {
az aks delete \
-g "kataCI" \
-n "$(_print_cluster_name)" \
--yes \
--no-wait
}
function main() {
action="${1:-}"
case "${action}" in
install-azure-cli) install_azure_cli ;;
login-azure) login_azure ;;
create-cluster) create_cluster ;;
install-bats) install_bats ;;
install-kubectl) install_kubectl ;;
get-cluster-credentials) get_cluster_credentials ;;
run-tests-aks) run_tests "aks" ;;
run-tests-sev) run_tests "sev" ;;
run-tests-snp) run_tests "snp" ;;
run-tests-tdx) run_tests "tdx" ;;
cleanup-sev) cleanup "sev" ;;
cleanup-snp) cleanup "snp" ;;
cleanup-tdx) cleanup "tdx" ;;
delete-cluster) delete_cluster ;;
*) >&2 echo "Invalid action"; exit 2 ;;
esac
}
main "$@"