diff --git a/.github/workflows/ci-coco-stability.yaml b/.github/workflows/ci-coco-stability.yaml deleted file mode 100644 index 3a75d91349..0000000000 --- a/.github/workflows/ci-coco-stability.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: Kata Containers CoCo Stability Tests Weekly -on: - # Note: This workload is not currently maintained, so skipping it's scheduled runs - # schedule: - # - cron: '0 0 * * 0' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -permissions: {} - -jobs: - kata-containers-ci-on-push: - permissions: - contents: read - packages: write - id-token: write - attestations: write - uses: ./.github/workflows/ci-weekly.yaml - with: - commit-hash: ${{ github.sha }} - pr-number: "weekly" - tag: ${{ github.sha }}-weekly - target-branch: ${{ github.ref_name }} - secrets: - AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }} - AZ_APPID: ${{ secrets.AZ_APPID }} - AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }} - AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }} - QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }} - KBUILD_SIGN_PIN: ${{ secrets.KBUILD_SIGN_PIN }} diff --git a/.github/workflows/ci-weekly.yaml b/.github/workflows/ci-weekly.yaml deleted file mode 100644 index 5497950a05..0000000000 --- a/.github/workflows/ci-weekly.yaml +++ /dev/null @@ -1,128 +0,0 @@ -name: Run the CoCo Kata Containers Stability CI -on: - workflow_call: - inputs: - commit-hash: - required: true - type: string - pr-number: - required: true - type: string - tag: - required: true - type: string - target-branch: - required: false - type: string - default: "" - secrets: - AUTHENTICATED_IMAGE_PASSWORD: - required: true - - AZ_APPID: - required: true - AZ_TENANT_ID: - required: true - AZ_SUBSCRIPTION_ID: - required: true - QUAY_DEPLOYER_PASSWORD: - required: true - KBUILD_SIGN_PIN: - required: true - -permissions: {} - -jobs: - build-kata-static-tarball-amd64: - permissions: - contents: read - packages: write - id-token: write - attestations: write - uses: ./.github/workflows/build-kata-static-tarball-amd64.yaml - with: - tarball-suffix: -${{ inputs.tag }} - commit-hash: ${{ inputs.commit-hash }} - target-branch: ${{ inputs.target-branch }} - secrets: - KBUILD_SIGN_PIN: ${{ secrets.KBUILD_SIGN_PIN }} - - publish-kata-deploy-payload-amd64: - needs: build-kata-static-tarball-amd64 - permissions: - contents: read - packages: write - uses: ./.github/workflows/publish-kata-deploy-payload.yaml - with: - tarball-suffix: -${{ inputs.tag }} - registry: ghcr.io - repo: ${{ github.repository_owner }}/kata-deploy-ci - tag: ${{ inputs.tag }}-amd64 - commit-hash: ${{ inputs.commit-hash }} - target-branch: ${{ inputs.target-branch }} - runner: ubuntu-22.04 - arch: amd64 - secrets: - QUAY_DEPLOYER_PASSWORD: ${{ secrets.QUAY_DEPLOYER_PASSWORD }} - - build-and-publish-tee-confidential-unencrypted-image: - name: build-and-publish-tee-confidential-unencrypted-image - permissions: - contents: read - packages: write - runs-on: ubuntu-22.04 - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: ${{ inputs.commit-hash }} - fetch-depth: 0 - persist-credentials: false - - - name: Rebase atop of the latest target branch - run: | - ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch" - env: - TARGET_BRANCH: ${{ inputs.target-branch }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - - - name: Login to Kata Containers ghcr.io - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker build and push - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.0.0 - with: - tags: ghcr.io/kata-containers/test-images:unencrypted-${{ inputs.pr-number }} - push: true - context: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/ - platforms: linux/amd64 - file: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/Dockerfile - - run-kata-coco-stability-tests: - needs: [publish-kata-deploy-payload-amd64, build-and-publish-tee-confidential-unencrypted-image] - uses: ./.github/workflows/run-kata-coco-stability-tests.yaml - with: - registry: ghcr.io - repo: ${{ github.repository_owner }}/kata-deploy-ci - tag: ${{ inputs.tag }}-amd64 - commit-hash: ${{ inputs.commit-hash }} - pr-number: ${{ inputs.pr-number }} - target-branch: ${{ inputs.target-branch }} - tarball-suffix: -${{ inputs.tag }} - secrets: - AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }} - AZ_APPID: ${{ secrets.AZ_APPID }} - AZ_TENANT_ID: ${{ secrets.AZ_TENANT_ID }} - AZ_SUBSCRIPTION_ID: ${{ secrets.AZ_SUBSCRIPTION_ID }} - permissions: - contents: read - id-token: write diff --git a/.github/workflows/run-kata-coco-stability-tests.yaml b/.github/workflows/run-kata-coco-stability-tests.yaml deleted file mode 100644 index e5c76215a2..0000000000 --- a/.github/workflows/run-kata-coco-stability-tests.yaml +++ /dev/null @@ -1,155 +0,0 @@ -name: CI | Run Kata CoCo k8s Stability Tests -on: - workflow_call: - inputs: - registry: - required: true - type: string - repo: - required: true - type: string - tag: - required: true - type: string - pr-number: - required: true - type: string - commit-hash: - required: false - type: string - target-branch: - required: false - type: string - default: "" - tarball-suffix: - required: false - type: string - secrets: - - AZ_APPID: - required: true - AZ_TENANT_ID: - required: true - AZ_SUBSCRIPTION_ID: - required: true - AUTHENTICATED_IMAGE_PASSWORD: - required: true - -permissions: {} - -jobs: - # Generate jobs for testing CoCo on non-TEE environments - run-stability-k8s-tests-coco-nontee: - name: run-stability-k8s-tests-coco-nontee - strategy: - fail-fast: false - matrix: - vmm: - - qemu-coco-dev - - qemu-coco-dev-runtime-rs - snapshotter: - - nydus - pull-type: - - guest-pull - runs-on: ubuntu-22.04 - permissions: - - id-token: write # Used for OIDC access to log into Azure - environment: - name: ci - deployment: false - env: - DOCKER_REGISTRY: ${{ inputs.registry }} - DOCKER_REPO: ${{ inputs.repo }} - DOCKER_TAG: ${{ inputs.tag }} - GH_PR_NUMBER: ${{ inputs.pr-number }} - KATA_HYPERVISOR: ${{ matrix.vmm }} - # Some tests rely on that variable to run (or not) - KBS: "true" - # Set the KBS ingress handler (empty string disables handling) - KBS_INGRESS: "aks" - KUBERNETES: "vanilla" - PULL_TYPE: ${{ matrix.pull-type }} - AUTHENTICATED_IMAGE_USER: ${{ vars.AUTHENTICATED_IMAGE_USER }} - AUTHENTICATED_IMAGE_PASSWORD: ${{ secrets.AUTHENTICATED_IMAGE_PASSWORD }} - SNAPSHOTTER: ${{ matrix.snapshotter }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: ${{ inputs.commit-hash }} - fetch-depth: 0 - persist-credentials: false - - - name: Rebase atop of the latest target branch - run: | - ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch" - env: - TARGET_BRANCH: ${{ inputs.target-branch }} - - - name: get-kata-tools-tarball - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - name: kata-tools-static-tarball-amd64${{ inputs.tarball-suffix }} - path: kata-tools-artifacts - - - name: Install kata-tools - run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts - - - name: Log into the Azure account - uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0 - with: - client-id: ${{ secrets.AZ_APPID }} - tenant-id: ${{ secrets.AZ_TENANT_ID }} - subscription-id: ${{ secrets.AZ_SUBSCRIPTION_ID }} - - - name: Create AKS cluster - uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 - with: - timeout_minutes: 15 - max_attempts: 20 - retry_on: error - retry_wait_seconds: 10 - command: bash tests/integration/kubernetes/gha-run.sh create-cluster - - - name: Install `bats` - run: bash tests/integration/kubernetes/gha-run.sh install-bats - - - name: Install `kubectl` - uses: azure/setup-kubectl@15650b3ad78fff148532a140b8a4c821796b2d7b # v5.0.0 - with: - version: 'latest' - - - name: Download credentials for the Kubernetes CLI to use them - run: bash tests/integration/kubernetes/gha-run.sh get-cluster-credentials - - - name: Deploy Kata - timeout-minutes: 10 - run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-aks - - - name: Deploy CoCo KBS - timeout-minutes: 10 - run: bash tests/integration/kubernetes/gha-run.sh deploy-coco-kbs - - - name: Install `kbs-client` - timeout-minutes: 10 - run: bash tests/integration/kubernetes/gha-run.sh install-kbs-client - - - name: Run stability tests - timeout-minutes: 300 - run: bash tests/stability/gha-stability-run.sh run-tests - - - name: Report tests - if: always() - run: bash tests/integration/kubernetes/gha-run.sh report-tests - - - name: Refresh OIDC token in case access token expired - if: always() - uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0 - with: - client-id: ${{ secrets.AZ_APPID }} - tenant-id: ${{ secrets.AZ_TENANT_ID }} - subscription-id: ${{ secrets.AZ_SUBSCRIPTION_ID }} - - - name: Delete AKS cluster - if: always() - run: bash tests/integration/kubernetes/gha-run.sh delete-cluster diff --git a/tests/stability/gha-stability-run.sh b/tests/stability/gha-stability-run.sh deleted file mode 100755 index 7bfce7c036..0000000000 --- a/tests/stability/gha-stability-run.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2024 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -set -o errexit -set -o nounset -set -o pipefail - -stability_dir="$(dirname "$(readlink -f "$0")")" -source "${stability_dir}/../metrics/lib/common.bash" -source "${stability_dir}/../gha-run-k8s-common.sh" -kata_tarball_dir="${2:-kata-artifacts}" - -function run_tests() { - info "Running scability test using ${KATA_HYPERVISOR} hypervisor" - bash "${stability_dir}/kubernetes_stability.sh" - - info "Running soak stability test using ${KATA_HYPERVISOR} hypervisor" - bash "${stability_dir}/kubernetes_soak_test.sh" - - info "Running stressng stability test using ${KATA_HYPERVISOR} hypervisor" - bash "${stability_dir}/kubernetes_stressng.sh" -} - -function main() { - action="${1:-}" - case "${action}" in - create-cluster) create_cluster ;; - install-bats) install_bats ;; - install-kata-tools) install_kata_tools "${2:-}" ;; - get-cluster-credentials) get_cluster_credentials ;; - deploy-kata-aks) deploy_kata "aks" ;; - deploy-coco-kbs) deploy_coco_kbs ;; - install-kbs-client) install_kbs_client ;; - run-tests) run_tests ;; - delete-cluster) cleanup "aks" ;; - *) >&2 die "Invalid argument" ;; - esac -} - -main "$@" diff --git a/tests/stability/kubernetes_soak_test.sh b/tests/stability/kubernetes_soak_test.sh deleted file mode 100755 index 93b1f20560..0000000000 --- a/tests/stability/kubernetes_soak_test.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2024 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -SCRIPT_PATH=$(dirname "$(readlink -f "$0")") -source "${SCRIPT_PATH}/../metrics/lib/common.bash" - -set -x - -replicas="${replicas:-8}" -deployment_name="${deployment_name:-deploymenttest}" -# How many times will we run the test loop... -iterations="${iterations:-10}" - -function delete_deployment() { - kubectl delete deployment "${deployment_name}" -} - -function go() { - kubectl scale deployment/"${deployment_name}" --replicas="${replicas}" - cmd="kubectl get deployment/${deployment_name} -o yaml | grep 'availableReplicas: ${replicas}'" - waitForProcess "300" "30" "${cmd}" -} - -function init() { - kubectl create -f "${SCRIPT_PATH}/runtimeclass_workloads/pod-deployment.yaml" - kubectl wait --for=condition=Available --timeout=100s deployment/"${deployment_name}" -} - -function main() { - check_processes - local i=0 - for (( i=1; i<="${iterations}"; i++ )); do - info "Start iteration $i of $iterations" - init - #spin them up - go - #shut them all down - delete_deployment - done -} - -main "$@" diff --git a/tests/stability/kubernetes_stability.sh b/tests/stability/kubernetes_stability.sh deleted file mode 100755 index 9a1e612533..0000000000 --- a/tests/stability/kubernetes_stability.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2024 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -set -o pipefail -set -x - -SCRIPT_PATH=$(dirname "$(readlink -f "$0")") -source "${SCRIPT_PATH}/../metrics/lib/common.bash" - -# Timeout is the duration of this test (seconds) -timeout=3600 -start_time=$(date +%s) -end_time=$((start_time+timeout)) - - -function main() { - # Check no processes are left behind - check_processes - - # Create pod - kubectl create -f "${SCRIPT_PATH}/runtimeclass_workloads/stability-test.yaml" - # Verify pod is running - pod_name="stability-test" - kubectl wait --for=condition=Ready --timeout=30s pod "${pod_name}" - - echo "Running kubernetes stability test" - count=0 - while [[ "${end_time}" -gt $(date +%s) ]]; do - echo "This is the number of iterations $count" - count=$((count+1)) - - cmd1="echo 'hello world' > file" - kubectl exec "${pod_name}" -- /bin/bash -c "${cmd1}" - - cmd2="rm -rf /file" - kubectl exec "${pod_name}" -- /bin/bash -c "${cmd2}" - - cmd3="touch /tmp/execWorks" - kubectl exec "${pod_name}" -- /bin/bash -c "${cmd3}" - - cmd4="ls /tmp | grep execWorks" - kubectl exec "${pod_name}" -- /bin/bash -c "${cmd4}" - - cmd5="rm -rf /tmp/execWorks" - kubectl exec "${pod_name}" -- /bin/bash -c "${cmd5}" - done - - kubectl delete -f "${SCRIPT_PATH}/runtimeclass_workloads/stability-test.yaml" -} - -main "$@" diff --git a/tests/stability/kubernetes_stressng.sh b/tests/stability/kubernetes_stressng.sh deleted file mode 100755 index 17e295335c..0000000000 --- a/tests/stability/kubernetes_stressng.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2024 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -set -o pipefail - -SCRIPT_PATH=$(dirname "$(readlink -f "$0")") -source "${SCRIPT_PATH}/../metrics/lib/common.bash" - -function main() { - # Check no processes are left behind - check_processes - # Create pod - kubectl create -f "${SCRIPT_PATH}/runtimeclass_workloads/stress-test.yaml" - # Verify pod is running - pod_name="stressng-test" - kubectl wait --for=condition=Ready --timeout=30s pod "${pod_name}" - - echo "Running stress matrix test" - cmd1="stress-ng --matrix 0 -t 90m" - kubectl exec "${pod_name}" -- /bin/bash -c "${cmd1}" - - echo "Running stress cpu test" - cmd2="stress-ng --cpu 0 --vm 2 -t 90m" - kubectl exec "${pod_name}" -- /bin/bash -c "${cmd2}" - - echo "Running stress io test" - cmd3="stress-ng --io 2 -t 90m" - kubectl exec "${pod_name}" -- /bin/bash -c "${cmd3}" - - kubectl delete -f "${SCRIPT_PATH}/runtimeclass_workloads/stress-test.yaml" - kubectl delete pod "${pod_name}" - check_processes -} - -main "$@"