From 028a97e0d5552153a4ae1469e4cf9715b1f1db25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 7 Sep 2023 12:26:47 +0200 Subject: [PATCH 01/10] ci: k8s: Use the proper command for sleep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `wait` waits for a job to complete, not a number of seconds. Not sure how I got that wrong in the first place, but it's what it's. Fixes: #6542 Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/gha-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 08c4b2f695..ff76df6466 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -144,7 +144,7 @@ function deploy_k3s() { curl -sfL https://get.k3s.io | sh - # This is an arbitrary value that came up from local tests - wait 240s + sleep 240s } function deploy_k8s() { From ad45ab5d337b8bf56d9ec2b8a203d9edab86fe2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 7 Sep 2023 13:02:47 +0200 Subject: [PATCH 02/10] ci: k8s: Ensure k3s is deploy with --write-kubeconfig-mode=644 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise the /etc/rancher/k3s/k3s.yaml is not readable by other users than root. As --write-config-mode is being passed, and that's an option that has to be passed to the `server`, -s is also added to the command line. Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/gha-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index ff76df6466..567d08282f 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -141,7 +141,7 @@ function deploy_kata() { } function deploy_k3s() { - curl -sfL https://get.k3s.io | sh - + curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644 # This is an arbitrary value that came up from local tests sleep 240s From ada65b988a33e743e9c043d419b5b7f177eca836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 7 Sep 2023 16:27:21 +0200 Subject: [PATCH 03/10] ci: k8s: Use vanilla kubectl with k3s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's download the vanilla kubectl binary into `/usr/bin/`, as we need to avoid hitting issues like: ```sh error: open /etc/rancher/k3s/k3s.yaml.lock: permission denied ``` The issue basically happens because k3s links `/usr/local/bin/kubectl` to `/usr/local/bin/k3s`, and that does extra stuff that vanilla `kubectl` doesn't do. Also, in order to properly use the k3s.yaml config with the vanilla kubectl, we're copying it to ~/.kube/config. Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/gha-run.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 567d08282f..8dca94a2c6 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -145,6 +145,27 @@ function deploy_k3s() { # This is an arbitrary value that came up from local tests sleep 240s + + # Download the kubectl binary into /usr/bin and remove /usr/local/bin/kubectl + # + # We need to do this to avoid hitting issues like: + # ```sh + # error: open /etc/rancher/k3s/k3s.yaml.lock: permission denied + # ``` + # Which happens basically because k3s links `/usr/local/bin/kubectl` + # to `/usr/local/bin/k3s`, and that does extra stuff that vanilla + # `kubectl` doesn't do. + ARCH=$(uname -m) + if [ "${ARCH}" = "x86_64" ]; then + ARCH=amd64 + fi + kubectl_version=$(/usr/local/bin/k3s kubectl version --short 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //' -e 's/\+k3s1//') + sudo curl -fL --progress-bar -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${kubectl_version}/bin/linux/${ARCH}/kubectl + sudo chmod +x /usr/bin/kubectl + sudo rm -rf /usr/local/bin/kubectl + + mkdir -p ~/.kube + cp /etc/rancher/k3s/k3s.yaml ~/.kube/config } function deploy_k8s() { From 369a8af8f74bfacac41ba57b5fdb42d90adf190c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 7 Sep 2023 19:49:52 +0200 Subject: [PATCH 04/10] ci: k8s: Decrease k3s sleep from 4 to 2 minutes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It should be plenty, and worked well in local tests. Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/gha-run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 8dca94a2c6..d896c872d3 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -144,7 +144,7 @@ function deploy_k3s() { curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644 # This is an arbitrary value that came up from local tests - sleep 240s + sleep 120s # Download the kubectl binary into /usr/bin and remove /usr/local/bin/kubectl # From 2df183fd9993696e98651f5e525ced67966e90d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 7 Sep 2023 21:47:29 +0200 Subject: [PATCH 05/10] ci: k8s: Append, instead of overwrite, the devmapper config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As we were using `tee` without the `-a` (or `--apend`) aptton, the containerd config would be overwritten, leading to a NotReady state of the Node. Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/gha-run.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index d896c872d3..af476acad4 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -59,7 +59,7 @@ EOF # We're not using this with baremetal machines, so we're fine on cutting # corners here and just append this to the configuration file. - cat<&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; esac + + sudo cat ${containerd_config_file} } function configure_snapshotter() { From 3de23034f8821cbb514b3ce261842ffae9793ad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Thu, 7 Sep 2023 23:46:58 +0200 Subject: [PATCH 06/10] ci: k8s: Wait some time after restarting k3s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's put a 1 minute sleep, just to make sure everything is back up again. Signed-off-by: Fabiano Fidêncio --- tests/integration/kubernetes/gha-run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index af476acad4..78c7f314fb 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -72,6 +72,7 @@ EOF *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; esac + sleep 60s sudo cat ${containerd_config_file} } From 8c9380a7980aa113ba91b626788b2036bb9ec919 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 8 Sep 2023 09:57:25 +0200 Subject: [PATCH 07/10] ci: k8s: Install bats on GARM runners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GARM runners do not come with the whole set of tools we need, or are used to when it comes to the GHA runners, so we need to manually install bats on those. Signed-off-by: Fabiano Fidêncio --- .github/workflows/run-k8s-tests-on-garm.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/run-k8s-tests-on-garm.yaml b/.github/workflows/run-k8s-tests-on-garm.yaml index eee9b48a34..2c90ec72b3 100644 --- a/.github/workflows/run-k8s-tests-on-garm.yaml +++ b/.github/workflows/run-k8s-tests-on-garm.yaml @@ -55,6 +55,9 @@ jobs: - name: Deploy Kata timeout-minutes: 10 run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx + + - name: Install `bats` + run: bash tests/integration/kubernetes/gha-run.sh install-bats - name: Run tests timeout-minutes: 30 From fa62a4c01b9ffac59bfe8740ee9c82e4c4f2dc81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 8 Sep 2023 10:02:08 +0200 Subject: [PATCH 08/10] ci: k8s: Export KUBERNETES env var MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So we have a better control on which flavour of kubernetes kata-deploy is expected to be targetting. Signed-off-by: Fabiano Fidêncio --- .github/workflows/run-k8s-tests-on-aks.yaml | 1 + .github/workflows/run-k8s-tests-on-sev.yaml | 1 + .github/workflows/run-k8s-tests-on-snp.yaml | 1 + .github/workflows/run-k8s-tests-on-tdx.yaml | 1 + tests/integration/kubernetes/gha-run.sh | 4 ++-- 5 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index 130be18296..261454bb2f 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -40,6 +40,7 @@ jobs: GH_PR_NUMBER: ${{ inputs.pr-number }} KATA_HOST_OS: ${{ matrix.host_os }} KATA_HYPERVISOR: ${{ matrix.vmm }} + KUBERNETES: "vanilla" USING_NFD: "false" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/run-k8s-tests-on-sev.yaml b/.github/workflows/run-k8s-tests-on-sev.yaml index bb4c5d0bdc..4260f43357 100644 --- a/.github/workflows/run-k8s-tests-on-sev.yaml +++ b/.github/workflows/run-k8s-tests-on-sev.yaml @@ -33,6 +33,7 @@ jobs: PR_NUMBER: ${{ inputs.pr-number }} KATA_HYPERVISOR: ${{ matrix.vmm }} KUBECONFIG: /home/kata/.kube/config + KUBERNETES: "vanilla" USING_NFD: "false" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/run-k8s-tests-on-snp.yaml b/.github/workflows/run-k8s-tests-on-snp.yaml index fde2131a7b..34cb5e6cba 100644 --- a/.github/workflows/run-k8s-tests-on-snp.yaml +++ b/.github/workflows/run-k8s-tests-on-snp.yaml @@ -33,6 +33,7 @@ jobs: PR_NUMBER: ${{ inputs.pr-number }} KATA_HYPERVISOR: ${{ matrix.vmm }} KUBECONFIG: /home/kata/.kube/config + KUBERNETES: "vanilla" USING_NFD: "false" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/run-k8s-tests-on-tdx.yaml b/.github/workflows/run-k8s-tests-on-tdx.yaml index 3cf2e9b10a..1e15dd5a61 100644 --- a/.github/workflows/run-k8s-tests-on-tdx.yaml +++ b/.github/workflows/run-k8s-tests-on-tdx.yaml @@ -32,6 +32,7 @@ jobs: DOCKER_TAG: ${{ inputs.tag }} PR_NUMBER: ${{ inputs.pr-number }} KATA_HYPERVISOR: ${{ matrix.vmm }} + KUBERNETES: "k3s" USING_NFD: "true" steps: - uses: actions/checkout@v3 diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 78c7f314fb..ff75091f02 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -118,7 +118,7 @@ function deploy_kata() { echo "::endgroup::" kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" - if [ "${platform}" = "tdx" ]; then + if [ "${KUBERNETES}" = "k3s" ]; then kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s" else kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" @@ -213,7 +213,7 @@ function cleanup() { kubectl config set-context --current --namespace=default kubectl delete namespace kata-containers-k8s-tests - if [ "${platform}" = "tdx" ]; then + if [ "${KUBERNETES}" = "k3s" ]; then deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s"" cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k3s"" else From 27fa7d828d2f5d35a5530b85e666ed58b785dc25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 8 Sep 2023 09:59:57 +0200 Subject: [PATCH 09/10] ci: k8s: Add a kata-deploy-garm target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We've been using the `kata-deploy-tdx` target as that also uses k3s as base, but it's better to just have a specific garm target. Signed-off-by: Fabiano Fidêncio --- .github/workflows/run-k8s-tests-on-garm.yaml | 4 ++-- tests/integration/kubernetes/gha-run.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-k8s-tests-on-garm.yaml b/.github/workflows/run-k8s-tests-on-garm.yaml index 2c90ec72b3..de5ed31cfd 100644 --- a/.github/workflows/run-k8s-tests-on-garm.yaml +++ b/.github/workflows/run-k8s-tests-on-garm.yaml @@ -54,7 +54,7 @@ jobs: - name: Deploy Kata timeout-minutes: 10 - run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-garm - name: Install `bats` run: bash tests/integration/kubernetes/gha-run.sh install-bats @@ -65,4 +65,4 @@ jobs: - name: Delete kata-deploy if: always() - run: bash tests/integration/kubernetes/gha-run.sh cleanup-tdx + run: bash tests/integration/kubernetes/gha-run.sh cleanup-garm diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index ff75091f02..9022bac7f6 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -261,6 +261,7 @@ function main() { deploy-kata-sev) deploy_kata "sev" ;; deploy-kata-snp) deploy_kata "snp" ;; deploy-kata-tdx) deploy_kata "tdx" ;; + deploy-kata-garm) deploy_kata "garm" ;; run-tests) run_tests ;; cleanup-sev) cleanup "sev" ;; cleanup-snp) cleanup "snp" ;; From aaec5a09f32574dacdabb95fd9fae079a084ead8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 8 Sep 2023 10:08:13 +0200 Subject: [PATCH 10/10] ci: k8s: devmapper tests should be using ubuntu 20.04 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit That's what we've been using as part of Jenkins, so let's ensure things will work as they did before, and only after that consider upgrading the base OS used for the tests. Signed-off-by: Fabiano Fidêncio --- .github/workflows/run-k8s-tests-on-garm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-k8s-tests-on-garm.yaml b/.github/workflows/run-k8s-tests-on-garm.yaml index de5ed31cfd..4588be335e 100644 --- a/.github/workflows/run-k8s-tests-on-garm.yaml +++ b/.github/workflows/run-k8s-tests-on-garm.yaml @@ -31,7 +31,7 @@ jobs: - devmapper k8s: - k3s - runs-on: garm-ubuntu-2204 + runs-on: garm-ubuntu-2004 env: DOCKER_REGISTRY: ${{ inputs.registry }} DOCKER_REPO: ${{ inputs.repo }}