mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-04-28 03:42:09 +00:00
ci: Use helm to deploy kata-deploy
Rather then modifying the kata-depoy scripts let's use Helm and create a values.yaml that can be used to render the final templates Signed-off-by: Zvonko Kaiser <zkaiser@nvidia.com>
This commit is contained in:
Zvonko Kaiser
Fabiano Fidêncio
parent
94b3348d3c
commit
51690bc157
@ -529,6 +529,29 @@ function ensure_yq() {
|
||||
hash -d yq 2> /dev/null || true # yq is preinstalled on GHA Ubuntu 22.04 runners so we clear Bash's PATH cache.
|
||||
}
|
||||
|
||||
function ensure_helm() {
|
||||
ensure_yq
|
||||
# The get-helm-3 script will take care of downloaading and installing Helm
|
||||
# properly on the system respecting ARCH, OS and other configurations.
|
||||
DESIRED_VERSION=$(get_from_kata_deps ".externals.helm.version")
|
||||
export DESIRED_VERSION
|
||||
|
||||
# Check if helm is available in the system's PATH
|
||||
if ! command -v helm &> /dev/null; then
|
||||
echo "Helm is not installed. Installing Helm..."
|
||||
curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
|
||||
# Verify the installation
|
||||
if command -v helm &> /dev/null; then
|
||||
echo "Helm installed successfully."
|
||||
else
|
||||
echo "Failed to install Helm."
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "Helm is already installed."
|
||||
fi
|
||||
}
|
||||
|
||||
# dependency: What we want to get the version from the versions.yaml file
|
||||
function get_from_kata_deps() {
|
||||
versions_file="${repo_root_dir}/versions.yaml"
|
||||
|
||||
@ -17,6 +17,7 @@ source "${kubernetes_dir}/../../gha-run-k8s-common.sh"
|
||||
source "${kubernetes_dir}/confidential_kbs.sh"
|
||||
# shellcheck disable=2154
|
||||
tools_dir="${repo_root_dir}/tools"
|
||||
helm_chart_dir="${tools_dir}/packaging/kata-deploy/helm-chart/kata-deploy"
|
||||
kata_tarball_dir="${2:-kata-artifacts}"
|
||||
|
||||
DOCKER_REGISTRY=${DOCKER_REGISTRY:-quay.io}
|
||||
@ -146,6 +147,7 @@ function deploy_coco_kbs() {
|
||||
|
||||
function deploy_kata() {
|
||||
platform="${1}"
|
||||
ensure_helm
|
||||
ensure_yq
|
||||
|
||||
[ "$platform" = "kcli" ] && \
|
||||
@ -157,82 +159,59 @@ function deploy_kata() {
|
||||
|
||||
set_default_cluster_namespace
|
||||
|
||||
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
local values_yaml
|
||||
values_yaml=$(mktemp /tmp/values_yaml.XXXXXX)
|
||||
|
||||
# Enable debug for Kata Containers
|
||||
yq -i \
|
||||
'.spec.template.spec.containers[0].env[1].value = "true"' \
|
||||
"${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
# Create the runtime class only for the shim that's being tested
|
||||
yq -i \
|
||||
".spec.template.spec.containers[0].env[2].value = \"${KATA_HYPERVISOR}\"" \
|
||||
"${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
# Set the tested hypervisor as the default `kata` shim
|
||||
yq -i \
|
||||
".spec.template.spec.containers[0].env[3].value = \"${KATA_HYPERVISOR}\"" \
|
||||
"${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
# Let the `kata-deploy` script take care of the runtime class creation / removal
|
||||
yq -i \
|
||||
'.spec.template.spec.containers[0].env[4].value = "true"' \
|
||||
"${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
# Let the `kata-deploy` create the default `kata` runtime class
|
||||
yq -i \
|
||||
'.spec.template.spec.containers[0].env[5].value = "true"' \
|
||||
"${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
# Enable 'default_vcpus' hypervisor annotation
|
||||
yq -i \
|
||||
'.spec.template.spec.containers[0].env[6].value = "default_vcpus"' \
|
||||
"${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
yq -i ".k8sDistribution = \"${KUBERNETES}\"" "${values_yaml}"
|
||||
yq -i ".image.reference = \"${DOCKER_REGISTRY}/${DOCKER_REPO}\"" "${values_yaml}"
|
||||
yq -i ".image.tag = \"${DOCKER_TAG}\"" "${values_yaml}"
|
||||
yq -i ".env.debug = \"true\"" "${values_yaml}"
|
||||
yq -i ".env.shims = \"${KATA_HYPERVISOR}\"" "${values_yaml}"
|
||||
yq -i ".env.defaultShim = \"${KATA_HYPERVISOR}\"" "${values_yaml}"
|
||||
yq -i ".env.createRuntimeClasses = \"true\"" "${values_yaml}"
|
||||
yq -i ".env.createDefaultRuntimeClass = \"true\"" "${values_yaml}"
|
||||
yq -i ".env.allowedHypervisorAnnotations = \"default_vcpus\"" "${values_yaml}"
|
||||
yq -i ".env.snapshotterHandlerMapping = \"\"" "${values_yaml}"
|
||||
yq -i ".env.agentHttpsProxy = \"\"" "${values_yaml}"
|
||||
yq -i ".env.agentNoProxy = \"\"" "${values_yaml}"
|
||||
yq -i ".env.pullTypeMapping = \"\"" "${values_yaml}"
|
||||
yq -i ".env.hostOS = \"\"" "${values_yaml}"
|
||||
|
||||
if [ -n "${SNAPSHOTTER}" ]; then
|
||||
yq -i \
|
||||
".spec.template.spec.containers[0].env[7].value = \"${KATA_HYPERVISOR}:${SNAPSHOTTER}\"" \
|
||||
"${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
yq -i ".env.snapshotterHandlerMapping = \"${KATA_HYPERVISOR}:${SNAPSHOTTER}\"" "${values_yaml}"
|
||||
fi
|
||||
|
||||
if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
|
||||
ALLOWED_HYPERVISOR_ANNOTATIONS="initrd kernel default_vcpus"
|
||||
HOST_OS=${KATA_HOST_OS}
|
||||
yq -i ".env.allowedHypervisorAnnotations = \"initrd kernel default_vcpus\"" "${values_yaml}"
|
||||
yq -i ".env.hostOS = \"${KATA_HOST_OS}\"" "${values_yaml}"
|
||||
fi
|
||||
|
||||
if [ "${KATA_HYPERVISOR}" = "qemu" ]; then
|
||||
yq -i \
|
||||
'.spec.template.spec.containers[0].env[6].value = "image initrd kernel default_vcpus"' \
|
||||
"${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
yq -i ".env.allowedHypervisorAnnotations = \"image initrd kernel default_vcpus\"" "${values_yaml}"
|
||||
fi
|
||||
|
||||
if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then
|
||||
yq -i \
|
||||
".spec.template.spec.containers[0].env[8].value = \"${HTTPS_PROXY}\"" \
|
||||
"${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
|
||||
yq -i \
|
||||
".spec.template.spec.containers[0].env[9].value = \"${NO_PROXY}\"" \
|
||||
"${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
yq -i ".env.agentHttpsProxy = \"${HTTPS_PROXY}\"" "${values_yaml}"
|
||||
yq -i ".env.agentNoProxy = \"${NO_PROXY}\"" "${values_yaml}"
|
||||
fi
|
||||
|
||||
# Set the PULL_TYPE_MAPPING
|
||||
if [ "${PULL_TYPE}" != "default" ]; then
|
||||
yq -i \
|
||||
".spec.template.spec.containers[0].env[10].value = \"${KATA_HYPERVISOR}:${PULL_TYPE}\"" \
|
||||
"${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
yq -i ".env.pullTypeMapping = \"${KATA_HYPERVISOR}:${PULL_TYPE}\"" "${values_yaml}"
|
||||
fi
|
||||
|
||||
echo "::group::Final kata-deploy.yaml that is used in the test"
|
||||
cat "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image"
|
||||
echo "::group::Final kata-deploy manifests used in the test"
|
||||
cat "${values_yaml}"
|
||||
helm template "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system
|
||||
[ "$(yq .image.reference ${values_yaml})" = "${DOCKER_REGISTRY}/${DOCKER_REPO}" ] || die "Failed to set image reference"
|
||||
[ "$(yq .image.tag ${values_yaml})" = "${DOCKER_TAG}" ] || die "Failed to set image tag"
|
||||
echo "::endgroup::"
|
||||
|
||||
kubectl_retry apply -f "${tools_dir}/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
|
||||
case "${KUBERNETES}" in
|
||||
k0s) kubectl_retry apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k0s" ;;
|
||||
k3s) kubectl_retry apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/k3s" ;;
|
||||
rke2) kubectl_retry apply -k "${tools_dir}/packaging/kata-deploy/kata-deploy/overlays/rke2" ;;
|
||||
*) kubectl_retry apply -f "${tools_dir}/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
esac
|
||||
|
||||
local cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\<Running\>'"
|
||||
waitForProcess "${KATA_DEPLOY_WAIT_TIMEOUT}" 10 "$cmd"
|
||||
# will wait until all Pods, PVCs, Services, and minimum number of Pods
|
||||
# of a Deployment, StatefulSet, or ReplicaSet are in a ready state
|
||||
# before marking the release as successful. It will wait for as long
|
||||
# as --timeout -- Ready >> Running
|
||||
helm install --wait --timeout 10m kata-deploy "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system
|
||||
|
||||
# This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
|
||||
# which may cause issues like not having the node properly labeled or the artefacts
|
||||
|
||||
@ -16,18 +16,3 @@ package: helm release
|
||||
|
||||
clean:
|
||||
rm kata-deploy-*.tgz
|
||||
|
||||
# go-get-tool will 'go get' any package $2 and install it to $1.
|
||||
PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
|
||||
define go-get-tool
|
||||
@[ -f $(PROJECT_DIR)/bin/$(1) ] || command -v $(1) >/dev/null 2>&1 || { \
|
||||
set -e ;\
|
||||
echo "Downloading $(2)" ;\
|
||||
GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
|
||||
rm -rf $$TMP_DIR ;\
|
||||
}
|
||||
endef
|
||||
|
||||
HELM = $(shell pwd)/bin/helm
|
||||
helm: ## Download helm locally if necessary.
|
||||
$(call go-get-tool,$(HELM),helm.sh/helm/v3/cmd/helm@latest)
|
||||
|
||||
@ -7,14 +7,14 @@
|
||||
Set the correct containerd conf path depending on the k8s distribution
|
||||
*/}}
|
||||
{{- define "containerdConfPath" -}}
|
||||
{{- if eq .k8sDistribution "k8s" -}}
|
||||
/etc/containerd/
|
||||
{{- else if eq .k8sDistribution "rke2" -}}
|
||||
{{- if eq .k8sDistribution "rke2" -}}
|
||||
/var/lib/rancher/rke2/agent/etc/containerd/
|
||||
{{- else if eq .k8sDistribution "k3s" -}}
|
||||
/var/lib/rancher/k3s/agent/etc/containerd/
|
||||
{{- else if eq .k8sDistribution "k0s" -}}
|
||||
/etc/k0s/containerd.d/
|
||||
{{- else -}}
|
||||
/etc/containerd/
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
|
||||
@ -218,6 +218,11 @@ externals:
|
||||
version: "1.36.1"
|
||||
url: "https://busybox.net/downloads"
|
||||
|
||||
helm:
|
||||
description: "Kubernetes package manager"
|
||||
url: "https://get.helm.sh/"
|
||||
version: "v3.15.2"
|
||||
|
||||
cni-plugins:
|
||||
description: "CNI network plugins"
|
||||
url: "https://github.com/containernetworking/plugins"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user