From f7976a40e48daee6434ad52901c42c7f62261a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 3 Mar 2025 16:28:49 +0100 Subject: [PATCH] tests: Create a helm_helper() common function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's use what we have in the k8s functional tests to create a common function to deploy kata containers using our helm charts. This will help us immensely in the kata-deploy testing side in the near future. Signed-off-by: Fabiano Fidêncio --- tests/gha-run-k8s-common.sh | 109 ++++++++++++++++++++++ tests/integration/kubernetes/gha-run.sh | 118 ++++++------------------ 2 files changed, 135 insertions(+), 92 deletions(-) diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh index 94da8fbd60..fb6105049a 100644 --- a/tests/gha-run-k8s-common.sh +++ b/tests/gha-run-k8s-common.sh @@ -7,6 +7,7 @@ tests_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${tests_dir}/common.bash" kubernetes_dir="${tests_dir}/integration/kubernetes" +helm_chart_dir="${repo_root_dir}/tools/packaging/kata-deploy/helm-chart/kata-deploy" AZ_APPID="${AZ_APPID:-}" AZ_PASSWORD="${AZ_PASSWORD:-}" @@ -14,6 +15,22 @@ AZ_SUBSCRIPTION_ID="${AZ_SUBSCRIPTION_ID:-}" AZ_TENANT_ID="${AZ_TENANT_ID:-}" GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-oci-distribution}" GH_PR_NUMBER="${GH_PR_NUMBER:-}" +HELM_DEFAULT_INSTALLATION="${HELM_DEFAULT_INSTALLATION:-false}" +HELM_AGENT_HTTPS_PROXY="${HELM_AGENT_HTTPS_PROXY:-}" +HELM_AGENT_NO_PROXY="${HELM_AGENT_NO_PROXY:-}" +HELM_ALLOWED_HYPERVISOR_ANNOTATIONS="${HELM_ALLOWED_HYPERVISOR_ANNOTATIONS:-}" +HELM_CREATE_RUNTIME_CLASSES="${HELM_CREATE_RUNTIME_CLASSES:-}" +HELM_CREATE_DEFAULT_RUNTIME_CLASS="${HELM_CREATE_DEFAULT_RUNTIME_CLASS:-}" +HELM_DEBUG="${HELM_DEBUG:-}" +HELM_DEFAULT_SHIM="${HELM_DEFAULT_SHIM:-}" +HELM_HOST_OS="${HELM_HOST_OS:-}" +HELM_IMAGE_REFERENCE="${HELM_IMAGE_REFERENCE:-}" +HELM_IMAGE_TAG="${HELM_IMAGE_TAG:-}" +HELM_K8S_DISTRIBUTION="${HELM_K8S_DISTRIBUTION:-}" +HELM_PULL_TYPE_MAPPING="${HELM_PULL_TYPE_MAPPING:-}" +HELM_SHIMS="${HELM_SHIMS:-}" +HELM_SNAPSHOTTER_HANDLER_MAPPING="${HELM_SNAPSHOTTER_HANDLER_MAPPING:-}" +KATA_DEPLOY_WAIT_TIMEOUT="${KATA_DEPLOY_WAIT_TIMEOUT:-600}" KATA_HOST_OS="${KATA_HOST_OS:-}" KUBERNETES="${KUBERNETES:-}" K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}" @@ -427,3 +444,95 @@ function delete_test_runners(){ fi done } + +function helm_helper() { + local max_tries + local interval + local i + local values_yaml + + ensure_yq + ensure_helm + + values_yaml=$(mktemp -t values_yaml.XXXXXX) + + if [[ -z "${HELM_IMAGE_REFERENCE}" ]]; then + die "HELM_IMAGE_REFERENCE environment variable cannot be empty." + fi + yq -i ".image.reference = \"${HELM_IMAGE_REFERENCE}\"" "${values_yaml}" + + if [[ -z "${HELM_IMAGE_TAG}" ]]; then + die "HELM_IMAGE_TAG environment variable cannot be empty." + fi + yq -i ".image.tag = \"${HELM_IMAGE_TAG}\"" "${values_yaml}" + + [[ -n "${HELM_K8S_DISTRIBUTION}" ]] && yq -i ".k8sDistribution = \"${HELM_K8S_DISTRIBUTION}\"" "${values_yaml}" + + if [[ "${HELM_DEFAULT_INSTALLATION}" = "false" ]]; then + [[ -n "${HELM_DEBUG}" ]] && yq -i ".env.debug = \"${HELM_DEBUG}\"" "${values_yaml}" + [[ -n "${HELM_SHIMS}" ]] && yq -i ".env.shims = \"${HELM_SHIMS}\"" "${values_yaml}" + [[ -n "${HELM_DEFAULT_SHIM}" ]] && yq -i ".env.defaultShim = \"${HELM_DEFAULT_SHIM}\"" "${values_yaml}" + [[ -n "${HELM_CREATE_RUNTIME_CLASSES}" ]] && yq -i ".env.createRuntimeClasses = \"${HELM_CREATE_RUNTIME_CLASSES}\"" "${values_yaml}" + [[ -n "${HELM_CREATE_DEFAULT_RUNTIME_CLASS}" ]] && yq -i ".env.createDefaultRuntimeClass = \"${HELM_CREATE_DEFAULT_RUNTIME_CLASS}\"" "${values_yaml}" + [[ -n "${HELM_ALLOWED_HYPERVISOR_ANNOTATIONS}" ]] && yq -i ".env.allowedHypervisorAnnotations = \"${HELM_ALLOWED_HYPERVISOR_ANNOTATIONS}\"" "${values_yaml}" + [[ -n "${HELM_SNAPSHOTTER_HANDLER_MAPPING}" ]] && yq -i ".env.snapshotterHandlerMapping = \"${HELM_SNAPSHOTTER_HANDLER_MAPPING}\"" "${values_yaml}" + [[ -n "${HELM_AGENT_HTTPS_PROXY}" ]] && yq -i ".env.agentHttpsProxy = \"${HELM_AGENT_HTTPS_PROXY}\"" "${values_yaml}" + [[ -n "${HELM_AGENT_NO_PROXY}" ]] && yq -i ".env.agentNoProxy = \"${HELM_AGENT_NO_PROXY}\"" "${values_yaml}" + [[ -n "${HELM_PULL_TYPE_MAPPING}" ]] && yq -i ".env.pullTypeMapping = \"${HELM_PULL_TYPE_MAPPING}\"" "${values_yaml}" + [[ -n "${HELM_HOST_OS}" ]] && yq -i ".env.hostOS=\"${HELM_HOST_OS}\"" "${values_yaml}" + fi + + echo "::group::Final kata-deploy manifests used in the test" + cat "${values_yaml}" + echo "" + helm template "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system + [[ "$(yq .image.reference "${values_yaml}")" = "${HELM_IMAGE_REFERENCE}" ]] || die "Failed to set image reference" + [[ "$(yq .image.tag "${values_yaml}")" = "${HELM_IMAGE_TAG}" ]] || die "Failed to set image tag" + echo "::endgroup::" + + max_tries=3 + interval=10 + i=10 + + # Retry loop for helm install to prevent transient failures due to instantly unreachable cluster + set +e # Disable immediate exit on failure + while true; do + helm upgrade --install kata-deploy "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system --debug + ret=${?} + if [[ ${ret} -eq 0 ]]; then + echo "Helm install succeeded!" + break + fi + i=$((i+1)) + if [[ ${i} -lt ${max_tries} ]]; then + echo "Retrying after ${interval} seconds (Attempt ${i} of $((max_tries - 1)))" + else + break + fi + sleep "${interval}" + done + set -e # Re-enable immediate exit on failure + if [[ ${i} -eq ${max_tries} ]]; then + die "Failed to deploy kata-deploy after ${max_tries} tries" + fi + + # `helm install --wait` does not take effect on single replicas and maxUnavailable=1 DaemonSets + # like kata-deploy on CI. So wait for pods being Running in the "traditional" way. + local cmd + cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\'" + waitForProcess "${KATA_DEPLOY_WAIT_TIMEOUT}" 10 "${cmd}" + + # FIXME: This is needed as the kata-deploy pod will be set to "Ready" + # when it starts running, which may cause issues like not having the + # node properly labeled or the artefacts properly deployed when the + # tests actually start running. + sleep 60s + + echo "::group::kata-deploy logs" + kubectl_retry -n kube-system logs --tail=100 -l name=kata-deploy + echo "::endgroup::" + + echo "::group::Runtime classes" + kubectl_retry get runtimeclass + echo "::endgroup::" +} diff --git a/tests/integration/kubernetes/gha-run.sh b/tests/integration/kubernetes/gha-run.sh index 3582d09c2f..28f419a676 100755 --- a/tests/integration/kubernetes/gha-run.sh +++ b/tests/integration/kubernetes/gha-run.sh @@ -17,13 +17,11 @@ source "${kubernetes_dir}/../../gha-run-k8s-common.sh" source "${kubernetes_dir}/confidential_kbs.sh" # shellcheck disable=2154 tools_dir="${repo_root_dir}/tools" -helm_chart_dir="${tools_dir}/packaging/kata-deploy/helm-chart/kata-deploy" kata_tarball_dir="${2:-kata-artifacts}" export DOCKER_REGISTRY="${DOCKER_REGISTRY:-quay.io}" export DOCKER_REPO="${DOCKER_REPO:-kata-containers/kata-deploy-ci}" export DOCKER_TAG="${DOCKER_TAG:-kata-containers-latest}" -export KATA_DEPLOY_WAIT_TIMEOUT="${KATA_DEPLOY_WAIT_TIMEOUT:-600}" export SNAPSHOTTER_DEPLOY_WAIT_TIMEOUT="${SNAPSHOTTER_DEPLOY_WAIT_TIMEOUT:-8m}" export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" export CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-containerd}" @@ -165,8 +163,6 @@ function deploy_coco_kbs() { function deploy_kata() { platform="${1:-}" - ensure_helm - ensure_yq [[ "${platform}" = "kcli" ]] && \ export KUBECONFIG="${HOME}/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig" @@ -177,106 +173,44 @@ function deploy_kata() { set_default_cluster_namespace - local values_yaml - values_yaml=$(mktemp /tmp/values_yaml.XXXXXX) - - yq -i ".k8sDistribution = \"${KUBERNETES}\"" "${values_yaml}" - yq -i ".image.reference = \"${DOCKER_REGISTRY}/${DOCKER_REPO}\"" "${values_yaml}" - yq -i ".image.tag = \"${DOCKER_TAG}\"" "${values_yaml}" - yq -i ".env.debug = \"true\"" "${values_yaml}" - yq -i ".env.shims = \"${KATA_HYPERVISOR}\"" "${values_yaml}" - yq -i ".env.defaultShim = \"${KATA_HYPERVISOR}\"" "${values_yaml}" - yq -i ".env.createRuntimeClasses = \"true\"" "${values_yaml}" - yq -i ".env.createDefaultRuntimeClass = \"true\"" "${values_yaml}" - yq -i ".env.allowedHypervisorAnnotations = \"default_vcpus\"" "${values_yaml}" - yq -i ".env.snapshotterHandlerMapping = \"\"" "${values_yaml}" - yq -i ".env.agentHttpsProxy = \"\"" "${values_yaml}" - yq -i ".env.agentNoProxy = \"\"" "${values_yaml}" - yq -i ".env.pullTypeMapping = \"\"" "${values_yaml}" - yq -i ".env.hostOS = \"\"" "${values_yaml}" - - if [[ -n "${SNAPSHOTTER}" ]]; then - yq -i ".env.snapshotterHandlerMapping = \"${KATA_HYPERVISOR}:${SNAPSHOTTER}\"" "${values_yaml}" - fi - + ANNOTATIONS="default_vcpus" if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then - yq -i ".env.allowedHypervisorAnnotations = \"image kernel default_vcpus\"" "${values_yaml}" - yq -i ".env.hostOS = \"${KATA_HOST_OS}\"" "${values_yaml}" + ANNOTATIONS="image kernel default_vcpus" fi - if [[ "${KATA_HYPERVISOR}" = "qemu" ]]; then - yq -i ".env.allowedHypervisorAnnotations = \"image initrd kernel default_vcpus\"" "${values_yaml}" + ANNOTATIONS="image initrd kernel default_vcpus" fi - if [[ "${KATA_HYPERVISOR}" = "qemu-tdx" ]]; then - yq -i ".env.agentHttpsProxy = \"${HTTPS_PROXY}\"" "${values_yaml}" - yq -i ".env.agentNoProxy = \"${NO_PROXY}\"" "${values_yaml}" + SNAPSHOTTER_HANDLER_MAPPING="" + if [[ -n "${SNAPSHOTTER}" ]]; then + SNAPSHOTTER_HANDLER_MAPPING="${KATA_HYPERVISOR}:${SNAPSHOTTER}" fi - # Set the PULL_TYPE_MAPPING + PULL_TYPE_MAPPING="" if [[ "${PULL_TYPE}" != "default" ]]; then - yq -i ".env.pullTypeMapping = \"${KATA_HYPERVISOR}:${PULL_TYPE}\"" "${values_yaml}" + PULL_TYPE_MAPPING="${KATA_HYPERVISOR}:${PULL_TYPE}" fi - echo "::group::Final kata-deploy manifests used in the test" - cat "${values_yaml}" - helm template "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system - [[ "$(yq .image.reference "${values_yaml}")" = "${DOCKER_REGISTRY}/${DOCKER_REPO}" ]] || die "Failed to set image reference" - [[ "$(yq .image.tag "${values_yaml}")" = "${DOCKER_TAG}" ]] || die "Failed to set image tag" - echo "::endgroup::" - - local max_tries - local interval - local i - - max_tries=3 - interval=10 - i=10 - - # Retry loop for helm install to prevent transient failures due to instantly unreachable cluster - set +e # Disable immediate exit on failure - while true; do - helm upgrade --install kata-deploy "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system --debug - ret=${?} - if [[ ${ret} -eq 0 ]]; then - echo "Helm install succeeded!" - break - fi - i=$((i+1)) - if [[ ${i} -lt ${max_tries} ]]; then - echo "Retrying after ${interval} seconds (Attempt ${i} of $((max_tries - 1)))" - else - break - fi - sleep "${interval}" - done - set -e # Re-enable immediate exit on failure - if [[ ${i} -eq ${max_tries} ]]; then - die "Failed to deploy kata-deploy after ${max_tries} tries" + HOST_OS="" + if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then + HOST_OS="${KATA_HOST_OS}" fi - # `helm install --wait` does not take effect on single replicas and maxUnavailable=1 DaemonSets - # like kata-deploy on CI. So wait for pods being Running in the "tradicional" way. - local cmd - cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\'" - waitForProcess "${KATA_DEPLOY_WAIT_TIMEOUT}" 10 "${cmd}" - - # This is needed as the kata-deploy pod will be set to "Ready" when it starts running, - # which may cause issues like not having the node properly labeled or the artefacts - # properly deployed when the tests actually start running. - if [[ "${platform}" = "aks" ]]; then - sleep 240s - else - sleep 60s - fi - - echo "::group::kata-deploy logs" - kubectl_retry -n kube-system logs --tail=100 -l name=kata-deploy - echo "::endgroup::" - - echo "::group::Runtime classes" - kubectl_retry get runtimeclass - echo "::endgroup::" + export HELM_K8S_DISTRIBUTION="${KUBERNETES}" + export HELM_IMAGE_REFERENCE="${DOCKER_REGISTRY}/${DOCKER_REPO}" + export HELM_IMAGE_TAG="${DOCKER_TAG}" + export HELM_DEBUG="true" + export HELM_SHIMS="${KATA_HYPERVISOR}" + export HELM_DEFAULT_SHIM="${KATA_HYPERVISOR}" + export HELM_CREATE_RUNTIME_CLASSES="true" + export HELM_CREATE_DEFAULT_RUNTIME_CLASS="true" + export HELM_ALLOWED_HYPERVISOR_ANNOTATIONS="${ANNOTATIONS}" + export HELM_SNAPSHOTTER_HANDLER_MAPPING="${SNAPSHOTTER_HANDLER_MAPPING}" + export HELM_AGENT_HTTPS_PROXY="${HTTPS_PROXY}" + export HELM_AGENT_NO_PROXY="${NO_PROXY}" + export HELM_PULL_TYPE_MAPPING="${PULL_TYPE_MAPPING}" + export HELM_HOST_OS="${HOST_OS}" + helm_helper } function install_kbs_client() {