diff --git a/.github/workflows/run-k8s-tests-on-aks.yaml b/.github/workflows/run-k8s-tests-on-aks.yaml index 130be18296..261454bb2f 100644 --- a/.github/workflows/run-k8s-tests-on-aks.yaml +++ b/.github/workflows/run-k8s-tests-on-aks.yaml @@ -40,6 +40,7 @@ jobs: GH_PR_NUMBER: ${{ inputs.pr-number }} KATA_HOST_OS: ${{ matrix.host_os }} KATA_HYPERVISOR: ${{ matrix.vmm }} + KUBERNETES: "vanilla" USING_NFD: "false" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/run-k8s-tests-on-garm.yaml b/.github/workflows/run-k8s-tests-on-garm.yaml index eee9b48a34..4588be335e 100644 --- a/.github/workflows/run-k8s-tests-on-garm.yaml +++ b/.github/workflows/run-k8s-tests-on-garm.yaml @@ -31,7 +31,7 @@ jobs: - devmapper k8s: - k3s - runs-on: garm-ubuntu-2204 + runs-on: garm-ubuntu-2004 env: DOCKER_REGISTRY: ${{ inputs.registry }} DOCKER_REPO: ${{ inputs.repo }} @@ -54,7 +54,10 @@ jobs: - name: Deploy Kata timeout-minutes: 10 - run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-tdx + run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-garm + + - name: Install `bats` + run: bash tests/integration/kubernetes/gha-run.sh install-bats - name: Run tests timeout-minutes: 30 @@ -62,4 +65,4 @@ jobs: - name: Delete kata-deploy if: always() - run: bash tests/integration/kubernetes/gha-run.sh cleanup-tdx + run: bash tests/integration/kubernetes/gha-run.sh cleanup-garm diff --git a/.github/workflows/run-k8s-tests-on-sev.yaml b/.github/workflows/run-k8s-tests-on-sev.yaml index bb4c5d0bdc..4260f43357 100644 --- a/.github/workflows/run-k8s-tests-on-sev.yaml +++ b/.github/workflows/run-k8s-tests-on-sev.yaml @@ -33,6 +33,7 @@ jobs: PR_NUMBER: ${{ inputs.pr-number }} KATA_HYPERVISOR: ${{ matrix.vmm }} KUBECONFIG: /home/kata/.kube/config + KUBERNETES: "vanilla" USING_NFD: "false" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/run-k8s-tests-on-snp.yaml b/.github/workflows/run-k8s-tests-on-snp.yaml index fde2131a7b..34cb5e6cba 100644 --- a/.github/workflows/run-k8s-tests-on-snp.yaml +++ b/.github/workflows/run-k8s-tests-on-snp.yaml @@ -33,6 +33,7 @@ jobs: PR_NUMBER: ${{ inputs.pr-number }} KATA_HYPERVISOR: ${{ matrix.vmm }} KUBECONFIG: /home/kata/.kube/config + KUBERNETES: "vanilla" USING_NFD: "false" steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/run-k8s-tests-on-tdx.yaml b/.github/workflows/run-k8s-tests-on-tdx.yaml index 3cf2e9b10a..1e15dd5a61 100644 --- a/.github/workflows/run-k8s-tests-on-tdx.yaml +++ b/.github/workflows/run-k8s-tests-on-tdx.yaml @@ -32,6 +32,7 @@ jobs: DOCKER_TAG: ${{ inputs.tag }} PR_NUMBER: ${{ inputs.pr-number }} KATA_HYPERVISOR: ${{ matrix.vmm }} + KUBERNETES: "k3s" USING_NFD: "true" steps: - uses: actions/checkout@v3 diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 08c4b2f695..9022bac7f6 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -59,7 +59,7 @@ EOF # We're not using this with baremetal machines, so we're fine on cutting # corners here and just append this to the configuration file. - cat<&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; esac + + sleep 60s + sudo cat ${containerd_config_file} } function configure_snapshotter() { @@ -115,7 +118,7 @@ function deploy_kata() { echo "::endgroup::" kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml" - if [ "${platform}" = "tdx" ]; then + if [ "${KUBERNETES}" = "k3s" ]; then kubectl apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s" else kubectl apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" @@ -141,10 +144,31 @@ function deploy_kata() { } function deploy_k3s() { - curl -sfL https://get.k3s.io | sh - + curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644 # This is an arbitrary value that came up from local tests - wait 240s + sleep 120s + + # Download the kubectl binary into /usr/bin and remove /usr/local/bin/kubectl + # + # We need to do this to avoid hitting issues like: + # ```sh + # error: open /etc/rancher/k3s/k3s.yaml.lock: permission denied + # ``` + # Which happens basically because k3s links `/usr/local/bin/kubectl` + # to `/usr/local/bin/k3s`, and that does extra stuff that vanilla + # `kubectl` doesn't do. + ARCH=$(uname -m) + if [ "${ARCH}" = "x86_64" ]; then + ARCH=amd64 + fi + kubectl_version=$(/usr/local/bin/k3s kubectl version --short 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //' -e 's/\+k3s1//') + sudo curl -fL --progress-bar -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${kubectl_version}/bin/linux/${ARCH}/kubectl + sudo chmod +x /usr/bin/kubectl + sudo rm -rf /usr/local/bin/kubectl + + mkdir -p ~/.kube + cp /etc/rancher/k3s/k3s.yaml ~/.kube/config } function deploy_k8s() { @@ -189,7 +213,7 @@ function cleanup() { kubectl config set-context --current --namespace=default kubectl delete namespace kata-containers-k8s-tests - if [ "${platform}" = "tdx" ]; then + if [ "${KUBERNETES}" = "k3s" ]; then deploy_spec="-k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s"" cleanup_spec="-k "${tools_dir}/packaging/kata-deploy/kata-cleanup/overlays/k3s"" else @@ -237,6 +261,7 @@ function main() { deploy-kata-sev) deploy_kata "sev" ;; deploy-kata-snp) deploy_kata "snp" ;; deploy-kata-tdx) deploy_kata "tdx" ;; + deploy-kata-garm) deploy_kata "garm" ;; run-tests) run_tests ;; cleanup-sev) cleanup "sev" ;; cleanup-snp) cleanup "snp" ;;