From dfb92e403e33b217815c209d6c9b7966c9ace31c Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 12 Jul 2024 15:08:43 -0300 Subject: [PATCH 1/5] tests/k8s: add "deploy-kata"/"cleanup" actions to gh-run.sh These new "kata-deploy" and "cleanup" actions are equivalent to "kata-deploy-garm" "cleanup-garm", respectively, and should be used on the workflows being migrated from GARM to Github's managed runners. Eventually "kata-deploy-garm" and "cleanup-garm" won't be used anymore then we will be able to remove them. See: #9940 Signed-off-by: Wainer dos Santos Moschetta --- tests/integration/kubernetes/gha-run.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 4df75441e8..fda4d98517 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -146,7 +146,7 @@ function deploy_coco_kbs() { } function deploy_kata() { - platform="${1}" + platform="${1:-}" ensure_helm ensure_yq @@ -389,7 +389,7 @@ function cleanup_kata_deploy() { } function cleanup() { - platform="${1}" + platform="${1:-}" test_type="${2:-k8s}" ensure_yq @@ -559,6 +559,7 @@ function main() { install-kbs-client) install_kbs_client ;; install-kubectl) install_kubectl ;; get-cluster-credentials) get_cluster_credentials ;; + deploy-kata) deploy_kata ;; deploy-kata-aks) deploy_kata "aks" ;; deploy-kata-kcli) deploy_kata "kcli" ;; deploy-kata-kubeadm) deploy_kata "kubeadm" ;; @@ -571,6 +572,7 @@ function main() { run-tests) run_tests ;; run-tests-kcli) run_tests "kcli" ;; collect-artifacts) collect_artifacts ;; + cleanup) cleanup ;; cleanup-kcli) cleanup "kcli" ;; cleanup-sev) cleanup "sev" ;; cleanup-snp) cleanup "snp" ;; From 3d053a70abeee686d054ea161411b1e43c88dc6f Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 12 Jul 2024 12:22:49 -0300 Subject: [PATCH 2/5] workflows: migrate run-k8s-tests-on-garm to free runners Switched to Github managed runners. The instance_type parameter was removed and K8S_TEST_HOST_TYPE is set to "all" which combine the tests of "small" and "normal". This way it will reduze to half of the jobs. See: #9940 Signed-off-by: Wainer dos Santos Moschetta --- .github/workflows/run-k8s-tests-on-garm.yaml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/run-k8s-tests-on-garm.yaml b/.github/workflows/run-k8s-tests-on-garm.yaml index 4736baa07f..9b8c4c4eea 100644 --- a/.github/workflows/run-k8s-tests-on-garm.yaml +++ b/.github/workflows/run-k8s-tests-on-garm.yaml @@ -1,4 +1,4 @@ -name: CI | Run kubernetes tests on GARM +name: CI | Run kubernetes tests on: workflow_call: inputs: @@ -38,13 +38,7 @@ jobs: k8s: - k3s instance: - - garm-ubuntu-2004 - - garm-ubuntu-2004-smaller - include: - - instance: garm-ubuntu-2004 - instance-type: normal - - instance: garm-ubuntu-2004-smaller - instance-type: small + - ubuntu-22.04 runs-on: ${{ matrix.instance }} env: DOCKER_REGISTRY: ${{ inputs.registry }} @@ -55,7 +49,7 @@ jobs: KUBERNETES: ${{ matrix.k8s }} SNAPSHOTTER: ${{ matrix.snapshotter }} USING_NFD: "false" - K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }} + K8S_TEST_HOST_TYPE: all steps: - uses: actions/checkout@v4 with: @@ -76,7 +70,7 @@ jobs: - name: Deploy Kata timeout-minutes: 10 - run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-garm + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata - name: Install `bats` run: bash tests/integration/kubernetes/gha-run.sh install-bats @@ -93,10 +87,10 @@ jobs: - name: Archive artifacts ${{ matrix.vmm }} uses: actions/upload-artifact@v4 with: - name: k8s-tests-garm-${{ matrix.vmm }}-${{ matrix.snapshotter }}-${{ matrix.k8s }}-${{ matrix.instance }}-${{ inputs.tag }} + name: k8s-tests-${{ matrix.vmm }}-${{ matrix.snapshotter }}-${{ matrix.k8s }}-${{ matrix.instance }}-${{ inputs.tag }} path: /tmp/artifacts retention-days: 1 - name: Delete kata-deploy if: always() - run: bash tests/integration/kubernetes/gha-run.sh cleanup-garm + run: bash tests/integration/kubernetes/gha-run.sh cleanup From ed0732c75d773c1608e711baff2ec89a9329ce43 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 12 Jul 2024 14:43:31 -0300 Subject: [PATCH 3/5] workflows: migrate run-k8s-tests-with-crio-on-garm to free runners Switch to Github managed runners just like the run-k8s-tests-on-garm workflow. See: #9940 Signed-off-by: Wainer dos Santos Moschetta --- .../run-k8s-tests-with-crio-on-garm.yaml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/run-k8s-tests-with-crio-on-garm.yaml b/.github/workflows/run-k8s-tests-with-crio-on-garm.yaml index 0370e1031b..e6cd8c139a 100644 --- a/.github/workflows/run-k8s-tests-with-crio-on-garm.yaml +++ b/.github/workflows/run-k8s-tests-with-crio-on-garm.yaml @@ -1,4 +1,4 @@ -name: CI | Run kubernetes tests, using CRI-O, on GARM +name: CI | Run kubernetes tests, using CRI-O on: workflow_call: inputs: @@ -32,13 +32,8 @@ jobs: k8s: - k0s instance: - - garm-ubuntu-2204 - - garm-ubuntu-2204-smaller + - ubuntu-22.04 include: - - instance: garm-ubuntu-2204 - instance-type: normal - - instance: garm-ubuntu-2204-smaller - instance-type: small - k8s: k0s k8s-extra-params: '--cri-socket remote:unix:///var/run/crio/crio.sock --kubelet-extra-args --cgroup-driver="systemd"' runs-on: ${{ matrix.instance }} @@ -51,7 +46,7 @@ jobs: KUBERNETES: ${{ matrix.k8s }} KUBERNETES_EXTRA_PARAMS: ${{ matrix.k8s-extra-params }} USING_NFD: "false" - K8S_TEST_HOST_TYPE: ${{ matrix.instance-type }} + K8S_TEST_HOST_TYPE: all steps: - uses: actions/checkout@v4 with: @@ -72,7 +67,7 @@ jobs: - name: Deploy Kata timeout-minutes: 10 - run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-garm + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata - name: Install `bats` run: bash tests/integration/kubernetes/gha-run.sh install-bats @@ -83,4 +78,4 @@ jobs: - name: Delete kata-deploy if: always() - run: bash tests/integration/kubernetes/gha-run.sh cleanup-garm + run: bash tests/integration/kubernetes/gha-run.sh cleanup From d11ce129ac3eff94dc74f7ea4085a3ca625f7db1 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 12 Jul 2024 16:40:23 -0300 Subject: [PATCH 4/5] workflows: merge run-k8s-tests-on-garm and run-k8s-tests-with-crio-on-garm Created the run-k8s-tests-on-amd64.yaml which is a merge of run-k8s-tests-on-garm.yaml and run-k8s-tests-with-crio-on-garm.yaml ps: renamed the job from 'run-k8s-tests' to 'run-k8s-tests-on-amd64' to it is easier to find on Github UI and be distinguished from s390x, ppc64le, etc... Signed-off-by: Wainer dos Santos Moschetta --- .github/workflows/ci.yaml | 16 +--- ...-garm.yaml => run-k8s-tests-on-amd64.yaml} | 18 ++++- .../run-k8s-tests-with-crio-on-garm.yaml | 81 ------------------- 3 files changed, 18 insertions(+), 97 deletions(-) rename .github/workflows/{run-k8s-tests-on-garm.yaml => run-k8s-tests-on-amd64.yaml} (80%) delete mode 100644 .github/workflows/run-k8s-tests-with-crio-on-garm.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fc230490bb..79feb3ce31 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -152,21 +152,9 @@ jobs: target-branch: ${{ inputs.target-branch }} secrets: inherit - run-k8s-tests-on-garm: + run-k8s-tests-on-amd64: needs: publish-kata-deploy-payload-amd64 - uses: ./.github/workflows/run-k8s-tests-on-garm.yaml - with: - registry: ghcr.io - repo: ${{ github.repository_owner }}/kata-deploy-ci - tag: ${{ inputs.tag }}-amd64 - commit-hash: ${{ inputs.commit-hash }} - pr-number: ${{ inputs.pr-number }} - target-branch: ${{ inputs.target-branch }} - secrets: inherit - - run-k8s-tests-with-crio-on-garm: - needs: publish-kata-deploy-payload-amd64 - uses: ./.github/workflows/run-k8s-tests-with-crio-on-garm.yaml + uses: ./.github/workflows/run-k8s-tests-on-amd64.yaml with: registry: ghcr.io repo: ${{ github.repository_owner }}/kata-deploy-ci diff --git a/.github/workflows/run-k8s-tests-on-garm.yaml b/.github/workflows/run-k8s-tests-on-amd64.yaml similarity index 80% rename from .github/workflows/run-k8s-tests-on-garm.yaml rename to .github/workflows/run-k8s-tests-on-amd64.yaml index 9b8c4c4eea..d069ee81e3 100644 --- a/.github/workflows/run-k8s-tests-on-garm.yaml +++ b/.github/workflows/run-k8s-tests-on-amd64.yaml @@ -1,4 +1,4 @@ -name: CI | Run kubernetes tests +name: CI | Run kubernetes tests on amd64 on: workflow_call: inputs: @@ -23,7 +23,7 @@ on: default: "" jobs: - run-k8s-tests: + run-k8s-tests-amd64: strategy: fail-fast: false matrix: @@ -33,12 +33,20 @@ jobs: - fc #firecracker - qemu - cloud-hypervisor + container_runtime: + - containerd snapshotter: - devmapper k8s: - k3s instance: - ubuntu-22.04 + include: + - vmm: qemu + container_runtime: crio + snapshotter: "" + instance: ubuntu-22.04 + k8s: k0s runs-on: ${{ matrix.instance }} env: DOCKER_REGISTRY: ${{ inputs.registry }} @@ -47,6 +55,7 @@ jobs: PR_NUMBER: ${{ inputs.pr-number }} KATA_HYPERVISOR: ${{ matrix.vmm }} KUBERNETES: ${{ matrix.k8s }} + KUBERNETES_EXTRA_PARAMS: ${{ matrix.container_runtime != 'crio' && '' || '--cri-socket remote:unix:///var/run/crio/crio.sock --kubelet-extra-args --cgroup-driver="systemd"' }} SNAPSHOTTER: ${{ matrix.snapshotter }} USING_NFD: "false" K8S_TEST_HOST_TYPE: all @@ -62,10 +71,15 @@ jobs: env: TARGET_BRANCH: ${{ inputs.target-branch }} + - name: Configure CRI-O + if: matrix.container_runtime == 'crio' + run: bash tests/integration/kubernetes/gha-run.sh setup-crio + - name: Deploy ${{ matrix.k8s }} run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s - name: Configure the ${{ matrix.snapshotter }} snapshotter + if: matrix.snapshotter != '' run: bash tests/integration/kubernetes/gha-run.sh configure-snapshotter - name: Deploy Kata diff --git a/.github/workflows/run-k8s-tests-with-crio-on-garm.yaml b/.github/workflows/run-k8s-tests-with-crio-on-garm.yaml deleted file mode 100644 index e6cd8c139a..0000000000 --- a/.github/workflows/run-k8s-tests-with-crio-on-garm.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: CI | Run kubernetes tests, using CRI-O -on: - workflow_call: - inputs: - registry: - required: true - type: string - repo: - required: true - type: string - tag: - required: true - type: string - pr-number: - required: true - type: string - commit-hash: - required: false - type: string - target-branch: - required: false - type: string - default: "" - -jobs: - run-k8s-tests: - strategy: - fail-fast: false - matrix: - vmm: - - qemu - k8s: - - k0s - instance: - - ubuntu-22.04 - include: - - k8s: k0s - k8s-extra-params: '--cri-socket remote:unix:///var/run/crio/crio.sock --kubelet-extra-args --cgroup-driver="systemd"' - runs-on: ${{ matrix.instance }} - env: - DOCKER_REGISTRY: ${{ inputs.registry }} - DOCKER_REPO: ${{ inputs.repo }} - DOCKER_TAG: ${{ inputs.tag }} - PR_NUMBER: ${{ inputs.pr-number }} - KATA_HYPERVISOR: ${{ matrix.vmm }} - KUBERNETES: ${{ matrix.k8s }} - KUBERNETES_EXTRA_PARAMS: ${{ matrix.k8s-extra-params }} - USING_NFD: "false" - K8S_TEST_HOST_TYPE: all - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.commit-hash }} - fetch-depth: 0 - - - name: Rebase atop of the latest target branch - run: | - ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch" - env: - TARGET_BRANCH: ${{ inputs.target-branch }} - - - name: Configure CRI-O - run: bash tests/integration/kubernetes/gha-run.sh setup-crio - - - name: Deploy ${{ matrix.k8s }} - run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s - - - name: Deploy Kata - timeout-minutes: 10 - run: bash tests/integration/kubernetes/gha-run.sh deploy-kata - - - name: Install `bats` - run: bash tests/integration/kubernetes/gha-run.sh install-bats - - - name: Run tests - timeout-minutes: 30 - run: bash tests/integration/kubernetes/gha-run.sh run-tests - - - name: Delete kata-deploy - if: always() - run: bash tests/integration/kubernetes/gha-run.sh cleanup From 374405aed189a9891bf0bdc1c847af08218d2c1f Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Wed, 7 Aug 2024 15:58:32 -0300 Subject: [PATCH 5/5] workflows/run-k8s-tests-on-amd64: remove 'instance' from matrix The jobs are all executed on ubuntu-22.04 so it's invariant and can be removed from the matrix (this will shrink the jobs names). Signed-off-by: Wainer dos Santos Moschetta --- .github/workflows/run-k8s-tests-on-amd64.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/run-k8s-tests-on-amd64.yaml b/.github/workflows/run-k8s-tests-on-amd64.yaml index d069ee81e3..338e923f29 100644 --- a/.github/workflows/run-k8s-tests-on-amd64.yaml +++ b/.github/workflows/run-k8s-tests-on-amd64.yaml @@ -39,15 +39,12 @@ jobs: - devmapper k8s: - k3s - instance: - - ubuntu-22.04 include: - vmm: qemu container_runtime: crio snapshotter: "" - instance: ubuntu-22.04 k8s: k0s - runs-on: ${{ matrix.instance }} + runs-on: ubuntu-22.04 env: DOCKER_REGISTRY: ${{ inputs.registry }} DOCKER_REPO: ${{ inputs.repo }}