diff --git a/.github/workflows/run-kata-deploy-tests-on-garm.yaml b/.github/workflows/run-kata-deploy-tests-on-garm.yaml index 4bfe660c4f..ce2ebf5403 100644 --- a/.github/workflows/run-kata-deploy-tests-on-garm.yaml +++ b/.github/workflows/run-kata-deploy-tests-on-garm.yaml @@ -34,6 +34,7 @@ jobs: - k0s - k3s - rke2 + - microk8s # TODO: There are a couple of vmm/k8s combination failing (https://github.com/kata-containers/kata-containers/issues/9854) # and we will put the entire kata-deploy-tests on GARM on maintenance. # TODO: Transition to free runner (see #9940). diff --git a/tests/functional/kata-deploy/kata-deploy.bats b/tests/functional/kata-deploy/kata-deploy.bats index 096df051fe..94b378d4d0 100644 --- a/tests/functional/kata-deploy/kata-deploy.bats +++ b/tests/functional/kata-deploy/kata-deploy.bats @@ -72,6 +72,8 @@ setup() { kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/k3s" elif [ "${KUBERNETES}" = "rke2" ]; then kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/rke2" + elif [ "${KUBERNETES}" = "microk8s" ]; then + kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/microk8s" else kubectl apply -f "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" fi @@ -142,6 +144,9 @@ teardown() { elif [ "${KUBERNETES}" = "rke2" ]; then deploy_spec="-k \"tools/packaging/kata-deploy/kata-deploy/overlays/rke2\"" cleanup_spec="-k \"tools/packaging/kata-deploy/kata-cleanup/overlays/rke2\"" + elif [ "${KUBERNETES}" = "microk8s" ]; then + deploy_spec="-k \"tools/packaging/kata-deploy/kata-deploy/overlays/microk8s\"" + cleanup_spec="-k \"tools/packaging/kata-deploy/kata-cleanup/overlays/microk8s\"" else deploy_spec="-f \"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml\"" cleanup_spec="-f \"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml\"" diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 4df12ee816..dcea96a264 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -323,6 +323,26 @@ function deploy_rke2() { sudo chown ${USER}:${USER} ~/.kube/config } +function deploy_microk8s() { + sudo snap install microk8s --classic + + # These are arbitrary values + sleep 30 + sudo /snap/bin/microk8s.status --wait-ready --timeout 300 + + # install kubectl + ARCH=$(arch_to_golang) + kubectl_version=$(/snap/bin/microk8s.version | grep -oe 'v[0-9]\+\(\.[0-9]\+\)*') + sudo curl -fL --progress-bar -o /usr/bin/kubectl https://dl.k8s.io/release/${kubectl_version}/bin/linux/${ARCH}/kubectl + sudo chmod +x /usr/bin/kubectl + sudo rm -rf /usr/local/bin/kubectl + + mkdir -p ~/.kube + sudo /snap/bin/microk8s.config > ~/.kube/config + sudo chown ${USER}:${USER} ~/.kube/config + newgrp microk8s +} + function _get_k0s_kubernetes_version_for_crio() { # k0s version will look like: # v1.27.5+k0s.0 @@ -358,6 +378,7 @@ function deploy_k8s() { k0s) deploy_k0s ;; k3s) deploy_k3s ;; rke2) deploy_rke2 ;; + microk8s) deploy_microk8s ;; *) >&2 echo "${KUBERNETES} flavour is not supported"; exit 2 ;; esac diff --git a/tools/packaging/kata-deploy/README.md b/tools/packaging/kata-deploy/README.md index c9ecbe4208..11ab7a1afc 100644 --- a/tools/packaging/kata-deploy/README.md +++ b/tools/packaging/kata-deploy/README.md @@ -77,6 +77,13 @@ $ sudo k0s kubectl apply -k kata-deploy/overlays/k0s $ sudo k0s kubectl apply -f kata-deploy/base/kata-deploy.yaml ``` +#### Microk8s Kubernetes cluster + +```bash +$ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml +$ kubectl apply -k https://github.com/kata-containers/kata-containers//tools/packaging/kata-deploy/kata-deploy/overlays/microk8s +``` + #### Vanilla Kubernetes cluster ```bash diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl index 426331e05d..d17332778c 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl @@ -13,6 +13,8 @@ Set the correct containerd conf path depending on the k8s distribution /var/lib/rancher/k3s/agent/etc/containerd/ {{- else if eq .k8sDistribution "k0s" -}} /etc/k0s/containerd.d/ +{{- else if eq .k8sDistribution "microk8s" -}} +/var/snap/microk8s/current/args/ {{- else -}} /etc/containerd/ {{- end -}} diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml b/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml index afda75d4c9..e1a2157e38 100644 --- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml +++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/values.yaml @@ -3,7 +3,7 @@ imagePullSecrets: [] image: reference: quay.io/kata-containers/kata-deploy tag: "" -# k8s-dist can be k8s, k3s, rke2, k0s +# k8s-dist can be k8s, k3s, rke2, k0s, microk8s k8sDistribution: "k8s" env: debug: "false" diff --git a/tools/packaging/kata-deploy/kata-cleanup/overlays/microk8s/kustomization.yaml b/tools/packaging/kata-deploy/kata-cleanup/overlays/microk8s/kustomization.yaml new file mode 100644 index 0000000000..fefc8daba6 --- /dev/null +++ b/tools/packaging/kata-deploy/kata-cleanup/overlays/microk8s/kustomization.yaml @@ -0,0 +1,7 @@ + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../base +patches: +- path: mount_microk8s_conf.yaml diff --git a/tools/packaging/kata-deploy/kata-cleanup/overlays/microk8s/mount_microk8s_conf.yaml b/tools/packaging/kata-deploy/kata-cleanup/overlays/microk8s/mount_microk8s_conf.yaml new file mode 100644 index 0000000000..db2ebe25af --- /dev/null +++ b/tools/packaging/kata-deploy/kata-cleanup/overlays/microk8s/mount_microk8s_conf.yaml @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kubelet-kata-cleanup + namespace: kube-system +spec: + template: + spec: + containers: + - name: kube-kata-cleanup + volumeMounts: + - name: containerd-conf + mountPath: /etc/containerd/ + volumes: + - name: containerd-conf + hostPath: + path: /var/snap/microk8s/current/args/ diff --git a/tools/packaging/kata-deploy/kata-deploy/overlays/microk8s/kustomization.yaml b/tools/packaging/kata-deploy/kata-deploy/overlays/microk8s/kustomization.yaml new file mode 100644 index 0000000000..fefc8daba6 --- /dev/null +++ b/tools/packaging/kata-deploy/kata-deploy/overlays/microk8s/kustomization.yaml @@ -0,0 +1,7 @@ + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../base +patches: +- path: mount_microk8s_conf.yaml diff --git a/tools/packaging/kata-deploy/kata-deploy/overlays/microk8s/mount_microk8s_conf.yaml b/tools/packaging/kata-deploy/kata-deploy/overlays/microk8s/mount_microk8s_conf.yaml new file mode 100644 index 0000000000..582dd74fd6 --- /dev/null +++ b/tools/packaging/kata-deploy/kata-deploy/overlays/microk8s/mount_microk8s_conf.yaml @@ -0,0 +1,12 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kata-deploy + namespace: kube-system +spec: + template: + spec: + volumes: + - name: containerd-conf + hostPath: + path: /var/snap/microk8s/current/args/ diff --git a/tools/packaging/kata-deploy/scripts/kata-deploy.sh b/tools/packaging/kata-deploy/scripts/kata-deploy.sh index c8e298338e..730beeac14 100755 --- a/tools/packaging/kata-deploy/scripts/kata-deploy.sh +++ b/tools/packaging/kata-deploy/scripts/kata-deploy.sh @@ -150,12 +150,15 @@ function delete_runtimeclasses() { function get_container_runtime() { local runtime=$(kubectl get node $NODE_NAME -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}') + local microk8s=$(kubectl get node $NODE_NAME -o jsonpath='{.metadata.labels.microk8s\.io\/cluster}') if [ "$?" -ne 0 ]; then die "invalid node name" fi if echo "$runtime" | grep -qE "cri-o"; then echo "cri-o" + elif [ "$microk8s" == "true" ]; then + echo "microk8s" elif echo "$runtime" | grep -qE 'containerd.*-k3s'; then if host_systemctl is-active --quiet rke2-agent; then echo "rke2-agent" @@ -193,6 +196,12 @@ function is_containerd_capable_of_using_drop_in_files() { return fi + if [ "$runtime" == "microk8s" ]; then + # microk8s use snap containerd + echo "false" + return + fi + local version_major=$(kubectl get node $NODE_NAME -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}' | grep -oE '[0-9]+\.[0-9]+' | cut -d'.' -f1) if [ $version_major -lt 2 ]; then # Only containerd 2.0 does the merge of the plugins section from different snippets, @@ -465,13 +474,15 @@ function configure_cri_runtime() { crio) configure_crio ;; - containerd | k3s | k3s-agent | rke2-agent | rke2-server | k0s-controller | k0s-worker) + containerd | k3s | k3s-agent | rke2-agent | rke2-server | k0s-controller | k0s-worker | microk8s) configure_containerd "$1" ;; esac if [ "$1" == "k0s-worker" ] || [ "$1" == "k0s-controller" ]; then # do nothing, k0s will automatically load the config on the fly : + elif [ "$1" == "microk8s" ]; then + host_systemctl restart snap.microk8s.daemon-containerd.service else host_systemctl daemon-reload host_systemctl restart "$1" @@ -658,6 +669,8 @@ function restart_cri_runtime() { if [ "${runtime}" == "k0s-worker" ] || [ "${runtime}" == "k0s-controller" ]; then # do nothing, k0s will automatically unload the config on the fly : + elif [ "$1" == "microk8s" ]; then + host_systemctl restart snap.microk8s.daemon-containerd.service else host_systemctl daemon-reload host_systemctl restart "${runtime}" @@ -669,7 +682,7 @@ function cleanup_cri_runtime() { crio) cleanup_crio ;; - containerd | k3s | k3s-agent | rke2-agent | rke2-server | k0s-controller | k0s-worker) + containerd | k3s | k3s-agent | rke2-agent | rke2-server | k0s-controller | k0s-worker | microk8s) cleanup_containerd ;; esac @@ -793,6 +806,9 @@ function main() { # CRI-O isn't consistent with the naming -- let's use crio to match the service file if [ "$runtime" == "cri-o" ]; then runtime="crio" + elif [ "$runtime" == "microk8s" ]; then + containerd_conf_file="/etc/containerd/containerd-template.toml" + containerd_conf_file_backup="${containerd_conf_file}.bak" elif [[ "$runtime" =~ ^(k3s|k3s-agent|rke2-agent|rke2-server)$ ]]; then containerd_conf_tmpl_file="${containerd_conf_file}.tmpl" containerd_conf_file_backup="${containerd_conf_tmpl_file}.bak" @@ -809,7 +825,7 @@ function main() { # only install / remove / update if we are dealing with CRIO or containerd - if [[ "$runtime" =~ ^(crio|containerd|k3s|k3s-agent|rke2-agent|rke2-server|k0s-worker|k0s-controller)$ ]]; then + if [[ "$runtime" =~ ^(crio|containerd|k3s|k3s-agent|rke2-agent|rke2-server|k0s-worker|k0s-controller|microk8s)$ ]]; then if [ "$runtime" != "crio" ]; then containerd_snapshotter_version_check snapshotter_handler_mapping_validation_check