mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-04-27 11:31:05 +00:00
tests: Create a helm_helper() common function
Let's use what we have in the k8s functional tests to create a common function to deploy kata containers using our helm charts. This will help us immensely in the kata-deploy testing side in the near future. Signed-off-by: Fabiano Fidêncio <fabiano@fidencio.org>
This commit is contained in:
Fabiano Fidêncio
parent
eb884d33a8
commit
f7976a40e4
@ -7,6 +7,7 @@
|
||||
tests_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "${tests_dir}/common.bash"
|
||||
kubernetes_dir="${tests_dir}/integration/kubernetes"
|
||||
helm_chart_dir="${repo_root_dir}/tools/packaging/kata-deploy/helm-chart/kata-deploy"
|
||||
|
||||
AZ_APPID="${AZ_APPID:-}"
|
||||
AZ_PASSWORD="${AZ_PASSWORD:-}"
|
||||
@ -14,6 +15,22 @@ AZ_SUBSCRIPTION_ID="${AZ_SUBSCRIPTION_ID:-}"
|
||||
AZ_TENANT_ID="${AZ_TENANT_ID:-}"
|
||||
GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-oci-distribution}"
|
||||
GH_PR_NUMBER="${GH_PR_NUMBER:-}"
|
||||
HELM_DEFAULT_INSTALLATION="${HELM_DEFAULT_INSTALLATION:-false}"
|
||||
HELM_AGENT_HTTPS_PROXY="${HELM_AGENT_HTTPS_PROXY:-}"
|
||||
HELM_AGENT_NO_PROXY="${HELM_AGENT_NO_PROXY:-}"
|
||||
HELM_ALLOWED_HYPERVISOR_ANNOTATIONS="${HELM_ALLOWED_HYPERVISOR_ANNOTATIONS:-}"
|
||||
HELM_CREATE_RUNTIME_CLASSES="${HELM_CREATE_RUNTIME_CLASSES:-}"
|
||||
HELM_CREATE_DEFAULT_RUNTIME_CLASS="${HELM_CREATE_DEFAULT_RUNTIME_CLASS:-}"
|
||||
HELM_DEBUG="${HELM_DEBUG:-}"
|
||||
HELM_DEFAULT_SHIM="${HELM_DEFAULT_SHIM:-}"
|
||||
HELM_HOST_OS="${HELM_HOST_OS:-}"
|
||||
HELM_IMAGE_REFERENCE="${HELM_IMAGE_REFERENCE:-}"
|
||||
HELM_IMAGE_TAG="${HELM_IMAGE_TAG:-}"
|
||||
HELM_K8S_DISTRIBUTION="${HELM_K8S_DISTRIBUTION:-}"
|
||||
HELM_PULL_TYPE_MAPPING="${HELM_PULL_TYPE_MAPPING:-}"
|
||||
HELM_SHIMS="${HELM_SHIMS:-}"
|
||||
HELM_SNAPSHOTTER_HANDLER_MAPPING="${HELM_SNAPSHOTTER_HANDLER_MAPPING:-}"
|
||||
KATA_DEPLOY_WAIT_TIMEOUT="${KATA_DEPLOY_WAIT_TIMEOUT:-600}"
|
||||
KATA_HOST_OS="${KATA_HOST_OS:-}"
|
||||
KUBERNETES="${KUBERNETES:-}"
|
||||
K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}"
|
||||
@ -427,3 +444,95 @@ function delete_test_runners(){
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function helm_helper() {
|
||||
local max_tries
|
||||
local interval
|
||||
local i
|
||||
local values_yaml
|
||||
|
||||
ensure_yq
|
||||
ensure_helm
|
||||
|
||||
values_yaml=$(mktemp -t values_yaml.XXXXXX)
|
||||
|
||||
if [[ -z "${HELM_IMAGE_REFERENCE}" ]]; then
|
||||
die "HELM_IMAGE_REFERENCE environment variable cannot be empty."
|
||||
fi
|
||||
yq -i ".image.reference = \"${HELM_IMAGE_REFERENCE}\"" "${values_yaml}"
|
||||
|
||||
if [[ -z "${HELM_IMAGE_TAG}" ]]; then
|
||||
die "HELM_IMAGE_TAG environment variable cannot be empty."
|
||||
fi
|
||||
yq -i ".image.tag = \"${HELM_IMAGE_TAG}\"" "${values_yaml}"
|
||||
|
||||
[[ -n "${HELM_K8S_DISTRIBUTION}" ]] && yq -i ".k8sDistribution = \"${HELM_K8S_DISTRIBUTION}\"" "${values_yaml}"
|
||||
|
||||
if [[ "${HELM_DEFAULT_INSTALLATION}" = "false" ]]; then
|
||||
[[ -n "${HELM_DEBUG}" ]] && yq -i ".env.debug = \"${HELM_DEBUG}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_SHIMS}" ]] && yq -i ".env.shims = \"${HELM_SHIMS}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_DEFAULT_SHIM}" ]] && yq -i ".env.defaultShim = \"${HELM_DEFAULT_SHIM}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_CREATE_RUNTIME_CLASSES}" ]] && yq -i ".env.createRuntimeClasses = \"${HELM_CREATE_RUNTIME_CLASSES}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_CREATE_DEFAULT_RUNTIME_CLASS}" ]] && yq -i ".env.createDefaultRuntimeClass = \"${HELM_CREATE_DEFAULT_RUNTIME_CLASS}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_ALLOWED_HYPERVISOR_ANNOTATIONS}" ]] && yq -i ".env.allowedHypervisorAnnotations = \"${HELM_ALLOWED_HYPERVISOR_ANNOTATIONS}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_SNAPSHOTTER_HANDLER_MAPPING}" ]] && yq -i ".env.snapshotterHandlerMapping = \"${HELM_SNAPSHOTTER_HANDLER_MAPPING}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_AGENT_HTTPS_PROXY}" ]] && yq -i ".env.agentHttpsProxy = \"${HELM_AGENT_HTTPS_PROXY}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_AGENT_NO_PROXY}" ]] && yq -i ".env.agentNoProxy = \"${HELM_AGENT_NO_PROXY}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_PULL_TYPE_MAPPING}" ]] && yq -i ".env.pullTypeMapping = \"${HELM_PULL_TYPE_MAPPING}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_HOST_OS}" ]] && yq -i ".env.hostOS=\"${HELM_HOST_OS}\"" "${values_yaml}"
|
||||
fi
|
||||
|
||||
echo "::group::Final kata-deploy manifests used in the test"
|
||||
cat "${values_yaml}"
|
||||
echo ""
|
||||
helm template "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system
|
||||
[[ "$(yq .image.reference "${values_yaml}")" = "${HELM_IMAGE_REFERENCE}" ]] || die "Failed to set image reference"
|
||||
[[ "$(yq .image.tag "${values_yaml}")" = "${HELM_IMAGE_TAG}" ]] || die "Failed to set image tag"
|
||||
echo "::endgroup::"
|
||||
|
||||
max_tries=3
|
||||
interval=10
|
||||
i=10
|
||||
|
||||
# Retry loop for helm install to prevent transient failures due to instantly unreachable cluster
|
||||
set +e # Disable immediate exit on failure
|
||||
while true; do
|
||||
helm upgrade --install kata-deploy "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system --debug
|
||||
ret=${?}
|
||||
if [[ ${ret} -eq 0 ]]; then
|
||||
echo "Helm install succeeded!"
|
||||
break
|
||||
fi
|
||||
i=$((i+1))
|
||||
if [[ ${i} -lt ${max_tries} ]]; then
|
||||
echo "Retrying after ${interval} seconds (Attempt ${i} of $((max_tries - 1)))"
|
||||
else
|
||||
break
|
||||
fi
|
||||
sleep "${interval}"
|
||||
done
|
||||
set -e # Re-enable immediate exit on failure
|
||||
if [[ ${i} -eq ${max_tries} ]]; then
|
||||
die "Failed to deploy kata-deploy after ${max_tries} tries"
|
||||
fi
|
||||
|
||||
# `helm install --wait` does not take effect on single replicas and maxUnavailable=1 DaemonSets
|
||||
# like kata-deploy on CI. So wait for pods being Running in the "traditional" way.
|
||||
local cmd
|
||||
cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\<Running\>'"
|
||||
waitForProcess "${KATA_DEPLOY_WAIT_TIMEOUT}" 10 "${cmd}"
|
||||
|
||||
# FIXME: This is needed as the kata-deploy pod will be set to "Ready"
|
||||
# when it starts running, which may cause issues like not having the
|
||||
# node properly labeled or the artefacts properly deployed when the
|
||||
# tests actually start running.
|
||||
sleep 60s
|
||||
|
||||
echo "::group::kata-deploy logs"
|
||||
kubectl_retry -n kube-system logs --tail=100 -l name=kata-deploy
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::Runtime classes"
|
||||
kubectl_retry get runtimeclass
|
||||
echo "::endgroup::"
|
||||
}
|
||||
|
||||
@ -17,13 +17,11 @@ source "${kubernetes_dir}/../../gha-run-k8s-common.sh"
|
||||
source "${kubernetes_dir}/confidential_kbs.sh"
|
||||
# shellcheck disable=2154
|
||||
tools_dir="${repo_root_dir}/tools"
|
||||
helm_chart_dir="${tools_dir}/packaging/kata-deploy/helm-chart/kata-deploy"
|
||||
kata_tarball_dir="${2:-kata-artifacts}"
|
||||
|
||||
export DOCKER_REGISTRY="${DOCKER_REGISTRY:-quay.io}"
|
||||
export DOCKER_REPO="${DOCKER_REPO:-kata-containers/kata-deploy-ci}"
|
||||
export DOCKER_TAG="${DOCKER_TAG:-kata-containers-latest}"
|
||||
export KATA_DEPLOY_WAIT_TIMEOUT="${KATA_DEPLOY_WAIT_TIMEOUT:-600}"
|
||||
export SNAPSHOTTER_DEPLOY_WAIT_TIMEOUT="${SNAPSHOTTER_DEPLOY_WAIT_TIMEOUT:-8m}"
|
||||
export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
|
||||
export CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-containerd}"
|
||||
@ -165,8 +163,6 @@ function deploy_coco_kbs() {
|
||||
|
||||
function deploy_kata() {
|
||||
platform="${1:-}"
|
||||
ensure_helm
|
||||
ensure_yq
|
||||
|
||||
[[ "${platform}" = "kcli" ]] && \
|
||||
export KUBECONFIG="${HOME}/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig"
|
||||
@ -177,106 +173,44 @@ function deploy_kata() {
|
||||
|
||||
set_default_cluster_namespace
|
||||
|
||||
local values_yaml
|
||||
values_yaml=$(mktemp /tmp/values_yaml.XXXXXX)
|
||||
|
||||
yq -i ".k8sDistribution = \"${KUBERNETES}\"" "${values_yaml}"
|
||||
yq -i ".image.reference = \"${DOCKER_REGISTRY}/${DOCKER_REPO}\"" "${values_yaml}"
|
||||
yq -i ".image.tag = \"${DOCKER_TAG}\"" "${values_yaml}"
|
||||
yq -i ".env.debug = \"true\"" "${values_yaml}"
|
||||
yq -i ".env.shims = \"${KATA_HYPERVISOR}\"" "${values_yaml}"
|
||||
yq -i ".env.defaultShim = \"${KATA_HYPERVISOR}\"" "${values_yaml}"
|
||||
yq -i ".env.createRuntimeClasses = \"true\"" "${values_yaml}"
|
||||
yq -i ".env.createDefaultRuntimeClass = \"true\"" "${values_yaml}"
|
||||
yq -i ".env.allowedHypervisorAnnotations = \"default_vcpus\"" "${values_yaml}"
|
||||
yq -i ".env.snapshotterHandlerMapping = \"\"" "${values_yaml}"
|
||||
yq -i ".env.agentHttpsProxy = \"\"" "${values_yaml}"
|
||||
yq -i ".env.agentNoProxy = \"\"" "${values_yaml}"
|
||||
yq -i ".env.pullTypeMapping = \"\"" "${values_yaml}"
|
||||
yq -i ".env.hostOS = \"\"" "${values_yaml}"
|
||||
|
||||
if [[ -n "${SNAPSHOTTER}" ]]; then
|
||||
yq -i ".env.snapshotterHandlerMapping = \"${KATA_HYPERVISOR}:${SNAPSHOTTER}\"" "${values_yaml}"
|
||||
fi
|
||||
|
||||
ANNOTATIONS="default_vcpus"
|
||||
if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
|
||||
yq -i ".env.allowedHypervisorAnnotations = \"image kernel default_vcpus\"" "${values_yaml}"
|
||||
yq -i ".env.hostOS = \"${KATA_HOST_OS}\"" "${values_yaml}"
|
||||
ANNOTATIONS="image kernel default_vcpus"
|
||||
fi
|
||||
|
||||
if [[ "${KATA_HYPERVISOR}" = "qemu" ]]; then
|
||||
yq -i ".env.allowedHypervisorAnnotations = \"image initrd kernel default_vcpus\"" "${values_yaml}"
|
||||
ANNOTATIONS="image initrd kernel default_vcpus"
|
||||
fi
|
||||
|
||||
if [[ "${KATA_HYPERVISOR}" = "qemu-tdx" ]]; then
|
||||
yq -i ".env.agentHttpsProxy = \"${HTTPS_PROXY}\"" "${values_yaml}"
|
||||
yq -i ".env.agentNoProxy = \"${NO_PROXY}\"" "${values_yaml}"
|
||||
SNAPSHOTTER_HANDLER_MAPPING=""
|
||||
if [[ -n "${SNAPSHOTTER}" ]]; then
|
||||
SNAPSHOTTER_HANDLER_MAPPING="${KATA_HYPERVISOR}:${SNAPSHOTTER}"
|
||||
fi
|
||||
|
||||
# Set the PULL_TYPE_MAPPING
|
||||
PULL_TYPE_MAPPING=""
|
||||
if [[ "${PULL_TYPE}" != "default" ]]; then
|
||||
yq -i ".env.pullTypeMapping = \"${KATA_HYPERVISOR}:${PULL_TYPE}\"" "${values_yaml}"
|
||||
PULL_TYPE_MAPPING="${KATA_HYPERVISOR}:${PULL_TYPE}"
|
||||
fi
|
||||
|
||||
echo "::group::Final kata-deploy manifests used in the test"
|
||||
cat "${values_yaml}"
|
||||
helm template "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system
|
||||
[[ "$(yq .image.reference "${values_yaml}")" = "${DOCKER_REGISTRY}/${DOCKER_REPO}" ]] || die "Failed to set image reference"
|
||||
[[ "$(yq .image.tag "${values_yaml}")" = "${DOCKER_TAG}" ]] || die "Failed to set image tag"
|
||||
echo "::endgroup::"
|
||||
|
||||
local max_tries
|
||||
local interval
|
||||
local i
|
||||
|
||||
max_tries=3
|
||||
interval=10
|
||||
i=10
|
||||
|
||||
# Retry loop for helm install to prevent transient failures due to instantly unreachable cluster
|
||||
set +e # Disable immediate exit on failure
|
||||
while true; do
|
||||
helm upgrade --install kata-deploy "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system --debug
|
||||
ret=${?}
|
||||
if [[ ${ret} -eq 0 ]]; then
|
||||
echo "Helm install succeeded!"
|
||||
break
|
||||
fi
|
||||
i=$((i+1))
|
||||
if [[ ${i} -lt ${max_tries} ]]; then
|
||||
echo "Retrying after ${interval} seconds (Attempt ${i} of $((max_tries - 1)))"
|
||||
else
|
||||
break
|
||||
fi
|
||||
sleep "${interval}"
|
||||
done
|
||||
set -e # Re-enable immediate exit on failure
|
||||
if [[ ${i} -eq ${max_tries} ]]; then
|
||||
die "Failed to deploy kata-deploy after ${max_tries} tries"
|
||||
HOST_OS=""
|
||||
if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
|
||||
HOST_OS="${KATA_HOST_OS}"
|
||||
fi
|
||||
|
||||
# `helm install --wait` does not take effect on single replicas and maxUnavailable=1 DaemonSets
|
||||
# like kata-deploy on CI. So wait for pods being Running in the "tradicional" way.
|
||||
local cmd
|
||||
cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\<Running\>'"
|
||||
waitForProcess "${KATA_DEPLOY_WAIT_TIMEOUT}" 10 "${cmd}"
|
||||
|
||||
# This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
|
||||
# which may cause issues like not having the node properly labeled or the artefacts
|
||||
# properly deployed when the tests actually start running.
|
||||
if [[ "${platform}" = "aks" ]]; then
|
||||
sleep 240s
|
||||
else
|
||||
sleep 60s
|
||||
fi
|
||||
|
||||
echo "::group::kata-deploy logs"
|
||||
kubectl_retry -n kube-system logs --tail=100 -l name=kata-deploy
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::Runtime classes"
|
||||
kubectl_retry get runtimeclass
|
||||
echo "::endgroup::"
|
||||
export HELM_K8S_DISTRIBUTION="${KUBERNETES}"
|
||||
export HELM_IMAGE_REFERENCE="${DOCKER_REGISTRY}/${DOCKER_REPO}"
|
||||
export HELM_IMAGE_TAG="${DOCKER_TAG}"
|
||||
export HELM_DEBUG="true"
|
||||
export HELM_SHIMS="${KATA_HYPERVISOR}"
|
||||
export HELM_DEFAULT_SHIM="${KATA_HYPERVISOR}"
|
||||
export HELM_CREATE_RUNTIME_CLASSES="true"
|
||||
export HELM_CREATE_DEFAULT_RUNTIME_CLASS="true"
|
||||
export HELM_ALLOWED_HYPERVISOR_ANNOTATIONS="${ANNOTATIONS}"
|
||||
export HELM_SNAPSHOTTER_HANDLER_MAPPING="${SNAPSHOTTER_HANDLER_MAPPING}"
|
||||
export HELM_AGENT_HTTPS_PROXY="${HTTPS_PROXY}"
|
||||
export HELM_AGENT_NO_PROXY="${NO_PROXY}"
|
||||
export HELM_PULL_TYPE_MAPPING="${PULL_TYPE_MAPPING}"
|
||||
export HELM_HOST_OS="${HOST_OS}"
|
||||
helm_helper
|
||||
}
|
||||
|
||||
function install_kbs_client() {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user