github/kata-containers
1
0
Fork 2
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-09-25 18:53:44 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #7896 from fidencio/topic/ground-work-for-testing-all-k8s-flavours-we-support

Browse Source 
ci: kata-deploy: Enable all k8s flavours that we support
This commit is contained in:
Fabiano Fidêncio
2023-09-19 17:44:34 +02:00
committed by GitHub
parent 6d5d844e5c f7fa7f602a
commit 5560e72024
8 changed files with 178 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
.github/workflows/ci.yaml vendored
View File

@@ -101,6 +101,18 @@ jobs:
target-branch: ${{ inputs.target-branch }}
secrets: inherit
run-kata-deploy-tests-on-garm:
needs: publish-kata-deploy-payload-amd64
uses: ./.github/workflows/run-kata-deploy-tests-on-garm.yaml
with:
registry: ghcr.io
repo: ${{ github.repository_owner }}/kata-deploy-ci
tag: ${{ inputs.tag }}-amd64
commit-hash: ${{ inputs.commit-hash }}
pr-number: ${{ inputs.pr-number }}
target-branch: ${{ inputs.target-branch }}
secrets: inherit
run-kata-deploy-tests-on-tdx:
needs: [publish-kata-deploy-payload-amd64, build-and-publish-tee-confidential-unencrypted-image]
uses: ./.github/workflows/run-kata-deploy-tests-on-tdx.yaml

1
.github/workflows/run-kata-deploy-tests-on-aks.yaml vendored
View File

@@ -44,6 +44,7 @@ jobs:
GH_PR_NUMBER: ${{ inputs.pr-number }}
KATA_HOST_OS: ${{ matrix.host_os }}
KATA_HYPERVISOR: ${{ matrix.vmm }}
KUBERNETES: "vanilla"
USING_NFD: "false"
steps:
- uses: actions/checkout@v3

65
.github/workflows/run-kata-deploy-tests-on-garm.yaml vendored Normal file
View File

@@ -0,0 +1,65 @@
name: CI | Run kata-deploy tests on GARM
on:
workflow_call:
inputs:
registry:
required: true
type: string
repo:
required: true
type: string
tag:
required: true
type: string
pr-number:
required: true
type: string
commit-hash:
required: false
type: string
target-branch:
required: false
type: string
default: ""
jobs:
run-kata-deploy-tests:
strategy:
fail-fast: false
matrix:
vmm:
- clh
- qemu
k8s:
- k0s
- k3s
- rke2
runs-on: garm-ubuntu-2004-small
env:
DOCKER_REGISTRY: ${{ inputs.registry }}
DOCKER_REPO: ${{ inputs.repo }}
DOCKER_TAG: ${{ inputs.tag }}
PR_NUMBER: ${{ inputs.pr-number }}
KATA_HYPERVISOR: ${{ matrix.vmm }}
KUBERNETES: ${{ matrix.k8s }}
USING_NFD: "false"
steps:
- uses: actions/checkout@v4
with:
ref: ${{ inputs.commit-hash }}
fetch-depth: 0
- name: Rebase atop of the latest target branch
run: |
./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
env:
TARGET_BRANCH: ${{ inputs.target-branch }}
- name: Deploy ${{ matrix.k8s }}
run: bash tests/functional/kata-deploy/gha-run.sh deploy-k8s
- name: Install `bats`
run: bash tests/functional/kata-deploy/gha-run.sh install-bats
- name: Run tests
run: bash tests/functional/kata-deploy/gha-run.sh run-tests

1
.github/workflows/run-kata-deploy-tests-on-tdx.yaml vendored
View File

@@ -36,6 +36,7 @@ jobs:
DOCKER_TAG: ${{ inputs.tag }}
PR_NUMBER: ${{ inputs.pr-number }}
KATA_HYPERVISOR: ${{ matrix.vmm }}
KUBERNETES: "k3s"
USING_NFD: "true"
steps:
- uses: actions/checkout@v3

1
tests/functional/kata-deploy/gha-run.sh
View File

@@ -56,6 +56,7 @@ function main() {
install-azure-cli) install_azure_cli ;;
login-azure) login_azure ;;
create-cluster) create_cluster "kata-deploy" ;;
deploy-k8s) deploy_k8s ;;
install-bats) install_bats ;;
install-kubectl) install_kubectl ;;
get-cluster-credentials) get_cluster_credentials "kata-deploy" ;;

22
tests/functional/kata-deploy/kata-deploy.bats
View File

@@ -48,8 +48,12 @@ setup() {
echo "::endgroup::"
kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
if [ "${platform}" = "tdx" ]; then
if [ "${KUBERNETES}" = "k0s" ]; then
kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s"
elif [ "${KUBERNETES}" = "k3s" ]; then
kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s"
elif [ "${KUBERNETES}" = "rke2" ]; then
kubectl apply -k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/rke2"
else
kubectl apply -f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
fi
@@ -74,12 +78,18 @@ setup() {
teardown() {
kubectl get runtimeclasses -o name | grep -v "kata-mshv-vm-isolation"
if [ "${platform}" = "tdx" ]; then
deploy_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s""
cleanup_spec="-k "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s""
if [ "${KUBERNETES}" = "k0s" ]; then
deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k0s\""
cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k0s\""
elif [ "${KUBERNETES}" = "k3s" ]; then
deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/k3s\""
cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/k3s\""
elif [ "${KUBERNETES}" = "rke2" ]; then
deploy_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/overlays/rke2\""
cleanup_spec="-k \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/overlays/rke2\""
else
deploy_spec="-f "${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml""
cleanup_spec="-f "${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml""
deploy_spec="-f \"${repo_root_dir}/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml\""
cleanup_spec="-f \"${repo_root_dir}/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml\""
fi
kubectl delete ${deploy_spec}

82
tests/gha-run-k8s-common.sh
View File

@@ -111,3 +111,85 @@ function get_nodes_and_pods_info() {
kubectl debug $(kubectl get nodes -o name) -it --image=quay.io/kata-containers/kata-debug:latest || true
kubectl get pods -o name | grep node-debugger | xargs kubectl delete || true
}
function deploy_k0s() {
curl -sSLf https://get.k0s.sh | sudo sh
sudo k0s install controller --single
sudo k0s start
# This is an arbitrary value that came up from local tests
sleep 120s
# Download the kubectl binary into /usr/bin so we can avoid depending
# on `k0s kubectl` command
ARCH=$(uname -m)
if [ "${ARCH}" = "x86_64" ]; then
ARCH=amd64
fi
kubectl_version=$(sudo k0s kubectl version --short 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //')
sudo curl -fL --progress-bar -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${kubectl_version}/bin/linux/${ARCH}/kubectl
sudo chmod +x /usr/bin/kubectl
mkdir -p ~/.kube
sudo cp /var/lib/k0s/pki/admin.conf ~/.kube/config
sudo chown ${USER}:${USER} ~/.kube/config
}
function deploy_k3s() {
curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644
# This is an arbitrary value that came up from local tests
sleep 120s
# Download the kubectl binary into /usr/bin and remove /usr/local/bin/kubectl
#
# We need to do this to avoid hitting issues like:
# ```sh
# error: open /etc/rancher/k3s/k3s.yaml.lock: permission denied
# ```
# Which happens basically because k3s links `/usr/local/bin/kubectl`
# to `/usr/local/bin/k3s`, and that does extra stuff that vanilla
# `kubectl` doesn't do.
ARCH=$(uname -m)
if [ "${ARCH}" = "x86_64" ]; then
ARCH=amd64
fi
kubectl_version=$(/usr/local/bin/k3s kubectl version --short 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //' -e 's/\+k3s1//')
sudo curl -fL --progress-bar -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${kubectl_version}/bin/linux/${ARCH}/kubectl
sudo chmod +x /usr/bin/kubectl
sudo rm -rf /usr/local/bin/kubectl
mkdir -p ~/.kube
cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
}
function deploy_rke2() {
curl -sfL https://get.rke2.io | sudo sh -
systemctl enable --now rke2-server.service
# This is an arbitrary value that came up from local tests
sleep 120s
# Link the kubectl binary into /usr/bin
sudo ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl
mkdir -p ~/.kube
sudo cp /etc/rancher/rke2/rke2.yaml ~/.kube/config
sudo chown ${USER}:${USER} ~/.kube/config
}
function deploy_k8s() {
echo "::group::Deploying ${KUBERNETES}"
case ${KUBERNETES} in
k0s) deploy_k0s ;;
k3s) deploy_k3s ;;
rke2) deploy_rke2 ;;
*) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;;
esac
echo "::endgroup::"
}

39
tests/integration/kubernetes/gha-run.sh
View File

@@ -143,45 +143,6 @@ function deploy_kata() {
echo "::endgroup::"
}
function deploy_k3s() {
curl -sfL https://get.k3s.io | sh -s - --write-kubeconfig-mode 644
# This is an arbitrary value that came up from local tests
sleep 120s
# Download the kubectl binary into /usr/bin and remove /usr/local/bin/kubectl
#
# We need to do this to avoid hitting issues like:
# ```sh
# error: open /etc/rancher/k3s/k3s.yaml.lock: permission denied
# ```
# Which happens basically because k3s links `/usr/local/bin/kubectl`
# to `/usr/local/bin/k3s`, and that does extra stuff that vanilla
# `kubectl` doesn't do.
ARCH=$(uname -m)
if [ "${ARCH}" = "x86_64" ]; then
ARCH=amd64
fi
kubectl_version=$(/usr/local/bin/k3s kubectl version --short 2>/dev/null | grep "Client Version" | sed -e 's/Client Version: //' -e 's/\+k3s1//')
sudo curl -fL --progress-bar -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${kubectl_version}/bin/linux/${ARCH}/kubectl
sudo chmod +x /usr/bin/kubectl
sudo rm -rf /usr/local/bin/kubectl
mkdir -p ~/.kube
cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
}
function deploy_k8s() {
echo "::group::Deploying ${KUBERNETES}"
case ${KUBERNETES} in
k3s) deploy_k3s ;;
*) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;;
esac
echo "::endgroup::"
}
function run_tests() {
# Delete any spurious tests namespace that was left behind
kubectl delete namespace kata-containers-k8s-tests &> /dev/null || true