From 51690bc157b950b94ecf8098059e522d03c57d51 Mon Sep 17 00:00:00 2001
From: Zvonko Kaiser <zkaiser@nvidia.com>
Date: Thu, 20 Jun 2024 17:11:07 +0000
Subject: [PATCH] ci: Use helm to deploy kata-deploy
Rather then modifying the kata-depoy scripts let's use Helm and
create a values.yaml that can be used to render the final templates
Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
---
tests/common.bash | 23 +++++
tests/integration/kubernetes/gha-run.sh | 91 +++++++------------
.../packaging/kata-deploy/helm-chart/Makefile | 15 ---
.../kata-deploy/templates/_helpers.tpl | 6 +-
versions.yaml | 5 +
5 files changed, 66 insertions(+), 74 deletions(-)
diff --git a/tests/common.bash b/tests/common.bash
index ba44fd89cd..8084f210c1 100644
--- a/tests/common.bash
+++ b/tests/common.bash
@@ -529,6 +529,29 @@ function ensure_yq() {
hash -d yq 2> /dev/null || true # yq is preinstalled on GHA Ubuntu 22.04 runners so we clear Bash's PATH cache.
}
+function ensure_helm() {
+ ensure_yq
+ # The get-helm-3 script will take care of downloaading and installing Helm
+ # properly on the system respecting ARCH, OS and other configurations.
+ DESIRED_VERSION=$(get_from_kata_deps ".externals.helm.version")
+ export DESIRED_VERSION
+
+ # Check if helm is available in the system's PATH
+ if ! command -v helm &> /dev/null; then
+ echo "Helm is not installed. Installing Helm..."
+ curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+ # Verify the installation
+ if command -v helm &> /dev/null; then
+ echo "Helm installed successfully."
+ else
+ echo "Failed to install Helm."
+ exit 1
+ fi
+ else
+ echo "Helm is already installed."
+ fi
+}
+
# dependency: What we want to get the version from the versions.yaml file
function get_from_kata_deps() {
versions_file="${repo_root_dir}/versions.yaml"
diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh
index cdef157687..9458c6a268 100755
--- a/tests/integration/kubernetes/gha-run.sh
+++ b/tests/integration/kubernetes/gha-run.sh
@@ -17,6 +17,7 @@ source "${kubernetes_dir}/../../gha-run-k8s-common.sh"
source "${kubernetes_dir}/confidential_kbs.sh"
# shellcheck disable=2154
tools_dir="${repo_root_dir}/tools"
+helm_chart_dir="${tools_dir}/packaging/kata-deploy/helm-chart/kata-deploy"
kata_tarball_dir="${2:-kata-artifacts}"
DOCKER_REGISTRY=${DOCKER_REGISTRY:-quay.io}
@@ -146,6 +147,7 @@ function deploy_coco_kbs() {
function deploy_kata() {
platform="${1}"
+ ensure_helm
ensure_yq
[ "$platform" = "kcli" ] && \
@@ -157,82 +159,59 @@ function deploy_kata() {
set_default_cluster_namespace
- sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+ local values_yaml
+ values_yaml=$(mktemp /tmp/values_yaml.XXXXXX)
- # Enable debug for Kata Containers
- yq -i \
- '.spec.template.spec.containers[0].env[1].value = "true"' \
- "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
- # Create the runtime class only for the shim that's being tested
- yq -i \
- ".spec.template.spec.containers[0].env[2].value = \"${KATA_HYPERVISOR}\"" \
- "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
- # Set the tested hypervisor as the default `kata` shim
- yq -i \
- ".spec.template.spec.containers[0].env[3].value = \"${KATA_HYPERVISOR}\"" \
- "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
- # Let the `kata-deploy` script take care of the runtime class creation / removal
- yq -i \
- '.spec.template.spec.containers[0].env[4].value = "true"' \
- "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
- # Let the `kata-deploy` create the default `kata` runtime class
- yq -i \
- '.spec.template.spec.containers[0].env[5].value = "true"' \
- "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
- # Enable 'default_vcpus' hypervisor annotation
- yq -i \
- '.spec.template.spec.containers[0].env[6].value = "default_vcpus"' \
- "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+ yq -i ".k8sDistribution = \"${KUBERNETES}\"" "${values_yaml}"
+ yq -i ".image.reference = \"${DOCKER_REGISTRY}/${DOCKER_REPO}\"" "${values_yaml}"
+ yq -i ".image.tag = \"${DOCKER_TAG}\"" "${values_yaml}"
+ yq -i ".env.debug = \"true\"" "${values_yaml}"
+ yq -i ".env.shims = \"${KATA_HYPERVISOR}\"" "${values_yaml}"
+ yq -i ".env.defaultShim = \"${KATA_HYPERVISOR}\"" "${values_yaml}"
+ yq -i ".env.createRuntimeClasses = \"true\"" "${values_yaml}"
+ yq -i ".env.createDefaultRuntimeClass = \"true\"" "${values_yaml}"
+ yq -i ".env.allowedHypervisorAnnotations = \"default_vcpus\"" "${values_yaml}"
+ yq -i ".env.snapshotterHandlerMapping = \"\"" "${values_yaml}"
+ yq -i ".env.agentHttpsProxy = \"\"" "${values_yaml}"
+ yq -i ".env.agentNoProxy = \"\"" "${values_yaml}"
+ yq -i ".env.pullTypeMapping = \"\"" "${values_yaml}"
+ yq -i ".env.hostOS = \"\"" "${values_yaml}"
if [ -n "${SNAPSHOTTER}" ]; then
- yq -i \
- ".spec.template.spec.containers[0].env[7].value = \"${KATA_HYPERVISOR}:${SNAPSHOTTER}\"" \
- "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+ yq -i ".env.snapshotterHandlerMapping = \"${KATA_HYPERVISOR}:${SNAPSHOTTER}\"" "${values_yaml}"
fi
if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
- ALLOWED_HYPERVISOR_ANNOTATIONS="initrd kernel default_vcpus"
- HOST_OS=${KATA_HOST_OS}
+ yq -i ".env.allowedHypervisorAnnotations = \"initrd kernel default_vcpus\"" "${values_yaml}"
+ yq -i ".env.hostOS = \"${KATA_HOST_OS}\"" "${values_yaml}"
fi
if [ "${KATA_HYPERVISOR}" = "qemu" ]; then
- yq -i \
- '.spec.template.spec.containers[0].env[6].value = "image initrd kernel default_vcpus"' \
- "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+ yq -i ".env.allowedHypervisorAnnotations = \"image initrd kernel default_vcpus\"" "${values_yaml}"
fi
if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then
- yq -i \
- ".spec.template.spec.containers[0].env[8].value = \"${HTTPS_PROXY}\"" \
- "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
-
- yq -i \
- ".spec.template.spec.containers[0].env[9].value = \"${NO_PROXY}\"" \
- "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+ yq -i ".env.agentHttpsProxy = \"${HTTPS_PROXY}\"" "${values_yaml}"
+ yq -i ".env.agentNoProxy = \"${NO_PROXY}\"" "${values_yaml}"
fi
# Set the PULL_TYPE_MAPPING
if [ "${PULL_TYPE}" != "default" ]; then
- yq -i \
- ".spec.template.spec.containers[0].env[10].value = \"${KATA_HYPERVISOR}:${PULL_TYPE}\"" \
- "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+ yq -i ".env.pullTypeMapping = \"${KATA_HYPERVISOR}:${PULL_TYPE}\"" "${values_yaml}"
fi
- echo "::group::Final kata-deploy.yaml that is used in the test"
- cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
- grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image"
+ echo "::group::Final kata-deploy manifests used in the test"
+ cat "${values_yaml}"
+ helm template "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system
+ [ "$(yq .image.reference ${values_yaml})" = "${DOCKER_REGISTRY}/${DOCKER_REPO}" ] || die "Failed to set image reference"
+ [ "$(yq .image.tag ${values_yaml})" = "${DOCKER_TAG}" ] || die "Failed to set image tag"
echo "::endgroup::"
- kubectl_retry apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
- case "${KUBERNETES}" in
- k0s) kubectl_retry apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k0s" ;;
- k3s) kubectl_retry apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s" ;;
- rke2) kubectl_retry apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/rke2" ;;
- *) kubectl_retry apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
- esac
-
- local cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\<Running\>'"
- waitForProcess "${KATA_DEPLOY_WAIT_TIMEOUT}" 10 "$cmd"
+ # will wait until all Pods, PVCs, Services, and minimum number of Pods
+ # of a Deployment, StatefulSet, or ReplicaSet are in a ready state
+ # before marking the release as successful. It will wait for as long
+ # as --timeout -- Ready >> Running
+ helm install --wait --timeout 10m kata-deploy "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system
# This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
# which may cause issues like not having the node properly labeled or the artefacts
diff --git a/tools/packaging/kata-deploy/helm-chart/Makefile b/tools/packaging/kata-deploy/helm-chart/Makefile
index 00a861bcd2..25f2939a6a 100644
--- a/tools/packaging/kata-deploy/helm-chart/Makefile
+++ b/tools/packaging/kata-deploy/helm-chart/Makefile
@@ -16,18 +16,3 @@ package: helm release
clean:
rm kata-deploy-*.tgz
-
-# go-get-tool will 'go get' any package $2 and install it to $1.
-PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
-define go-get-tool
-@[ -f $(PROJECT_DIR)/bin/$(1) ] || command -v $(1) >/dev/null 2>&1 || { \
-set -e ;\
-echo "Downloading $(2)" ;\
-GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
-rm -rf $$TMP_DIR ;\
-}
-endef
-
-HELM = $(shell pwd)/bin/helm
-helm: ## Download helm locally if necessary.
- $(call go-get-tool,$(HELM),helm.sh/helm/v3/cmd/helm@latest)
diff --git a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl
index 516b3b7056..426331e05d 100644
--- a/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl
+++ b/tools/packaging/kata-deploy/helm-chart/kata-deploy/templates/_helpers.tpl
@@ -7,14 +7,14 @@
Set the correct containerd conf path depending on the k8s distribution
*/}}
{{- define "containerdConfPath" -}}
-{{- if eq .k8sDistribution "k8s" -}}
-/etc/containerd/
-{{- else if eq .k8sDistribution "rke2" -}}
+{{- if eq .k8sDistribution "rke2" -}}
/var/lib/rancher/rke2/agent/etc/containerd/
{{- else if eq .k8sDistribution "k3s" -}}
/var/lib/rancher/k3s/agent/etc/containerd/
{{- else if eq .k8sDistribution "k0s" -}}
/etc/k0s/containerd.d/
+{{- else -}}
+/etc/containerd/
{{- end -}}
{{- end -}}
diff --git a/versions.yaml b/versions.yaml
index dc0709aafe..4c3ea64ed8 100644
--- a/versions.yaml
+++ b/versions.yaml
@@ -218,6 +218,11 @@ externals:
version: "1.36.1"
url: "https://busybox.net/downloads"
+ helm:
+ description: "Kubernetes package manager"
+ url: "https://get.helm.sh/"
+ version: "v3.15.2"
+
cni-plugins:
description: "CNI network plugins"
url: "https://github.com/containernetworking/plugins"