mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-04-28 03:42:09 +00:00
Merge pull request #10984 from fidencio/topic/tests-kata-deploy-ground-work-to-rewrite-the-tests
tests: kata-deploy: The rest of the ground work to rewrite the kata-deploy tests
This commit is contained in:
Fabiano Fidêncio
GitHub
commit
7a0ac55f22
@ -6,9 +6,10 @@
|
||||
#
|
||||
|
||||
load "${BATS_TEST_DIRNAME}/../../common.bash"
|
||||
repo_root_dir="${BATS_TEST_DIRNAME}/../../../"
|
||||
load "${repo_root_dir}/tests/gha-run-k8s-common.sh"
|
||||
|
||||
setup() {
|
||||
repo_root_dir="${BATS_TEST_DIRNAME}/../../../"
|
||||
ensure_yq
|
||||
|
||||
pushd "${repo_root_dir}"
|
||||
@ -25,77 +26,35 @@ setup() {
|
||||
"kata-${KATA_HYPERVISOR}\s+kata-${KATA_HYPERVISOR}" \
|
||||
)
|
||||
|
||||
|
||||
# Set the latest image, the one generated as part of the PR, to be used as part of the tests
|
||||
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
export HELM_IMAGE_REFERENCE="${DOCKER_REGISTRY}/${DOCKER_REPO}"
|
||||
export HELM_IMAGE_TAG="${DOCKER_TAG}"
|
||||
|
||||
# Enable debug for Kata Containers
|
||||
yq -i \
|
||||
'.spec.template.spec.containers[0].env[1].value = "true"' \
|
||||
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
export HELM_DEBUG="true"
|
||||
|
||||
# Create the runtime class only for the shim that's being tested
|
||||
yq -i \
|
||||
".spec.template.spec.containers[0].env[2].value = \"${KATA_HYPERVISOR}\"" \
|
||||
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
export HELM_SHIMS="${KATA_HYPERVISOR}"
|
||||
|
||||
# Set the tested hypervisor as the default `kata` shim
|
||||
yq -i \
|
||||
".spec.template.spec.containers[0].env[3].value = \"${KATA_HYPERVISOR}\"" \
|
||||
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
export HELM_DEFAULT_SHIM="${KATA_HYPERVISOR}"
|
||||
|
||||
# Let the `kata-deploy` script take care of the runtime class creation / removal
|
||||
yq -i \
|
||||
'.spec.template.spec.containers[0].env[4].value = "true"' \
|
||||
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
export HELM_CREATE_RUNTIME_CLASSES="true"
|
||||
|
||||
# Let the `kata-deploy` create the default `kata` runtime class
|
||||
yq -i \
|
||||
'.spec.template.spec.containers[0].env[5].value = "true"' \
|
||||
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
export HELM_CREATE_DEFAULT_RUNTIME_CLASS="true"
|
||||
|
||||
if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
|
||||
yq -i \
|
||||
".spec.template.spec.containers[0].env += [{\"name\": \"HOST_OS\", \"value\": \"${KATA_HOST_OS}\"}]" \
|
||||
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
fi
|
||||
HOST_OS=""
|
||||
if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
|
||||
HOST_OS="${KATA_HOST_OS}"
|
||||
fi
|
||||
export HELM_HOST_OS="${HOST_OS}"
|
||||
|
||||
echo "::group::Final kata-deploy.yaml that is used in the test"
|
||||
cat "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" || die "Failed to setup the tests image"
|
||||
echo "::endgroup::"
|
||||
export HELM_K8S_DISTRIBUTION="${KUBERNETES}"
|
||||
|
||||
echo "::group::Debug overlays directory content"
|
||||
echo "Current working directory: $(pwd)"
|
||||
ls -la tools/packaging/kata-deploy/kata-deploy/overlays/
|
||||
echo "::endgroup::"
|
||||
|
||||
kubectl apply -f "tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
|
||||
if [ "${KUBERNETES}" = "k0s" ]; then
|
||||
kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/k0s"
|
||||
elif [ "${KUBERNETES}" = "k3s" ]; then
|
||||
kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/k3s"
|
||||
elif [ "${KUBERNETES}" = "rke2" ]; then
|
||||
kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/rke2"
|
||||
elif [ "${KUBERNETES}" = "microk8s" ]; then
|
||||
kubectl apply -k "tools/packaging/kata-deploy/kata-deploy/overlays/microk8s"
|
||||
else
|
||||
kubectl apply -f "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
fi
|
||||
|
||||
local cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\<Running\>'"
|
||||
|
||||
if ! waitForProcess 240 10 "$cmd"; then
|
||||
echo "Kata-deploy pod is not running. Printing pod details for debugging:"
|
||||
kubectl -n kube-system get pods -o wide
|
||||
kubectl -n kube-system get pods -l name=kata-deploy -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | while read -r pod; do
|
||||
echo "Describing pod: $pod"
|
||||
kubectl -n kube-system describe pod "$pod"
|
||||
done
|
||||
|
||||
echo "ERROR: kata-deploy pod is not running, tests will not be execute."
|
||||
echo "ERROR: setup() aborting tests..."
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Give some time for the pod to finish what's doing and have the
|
||||
# runtimeclasses properly created
|
||||
sleep 30s
|
||||
helm_helper
|
||||
|
||||
echo "::group::kata-deploy logs"
|
||||
kubectl -n kube-system logs --tail=100 -l name=kata-deploy
|
||||
@ -135,52 +94,8 @@ setup() {
|
||||
teardown() {
|
||||
pushd "${repo_root_dir}"
|
||||
|
||||
if [ "${KUBERNETES}" = "k0s" ]; then
|
||||
deploy_spec="-k \"tools/packaging/kata-deploy/kata-deploy/overlays/k0s\""
|
||||
cleanup_spec="-k \"tools/packaging/kata-deploy/kata-cleanup/overlays/k0s\""
|
||||
elif [ "${KUBERNETES}" = "k3s" ]; then
|
||||
deploy_spec="-k \"tools/packaging/kata-deploy/kata-deploy/overlays/k3s\""
|
||||
cleanup_spec="-k \"tools/packaging/kata-deploy/kata-cleanup/overlays/k3s\""
|
||||
elif [ "${KUBERNETES}" = "rke2" ]; then
|
||||
deploy_spec="-k \"tools/packaging/kata-deploy/kata-deploy/overlays/rke2\""
|
||||
cleanup_spec="-k \"tools/packaging/kata-deploy/kata-cleanup/overlays/rke2\""
|
||||
elif [ "${KUBERNETES}" = "microk8s" ]; then
|
||||
deploy_spec="-k \"tools/packaging/kata-deploy/kata-deploy/overlays/microk8s\""
|
||||
cleanup_spec="-k \"tools/packaging/kata-deploy/kata-cleanup/overlays/microk8s\""
|
||||
else
|
||||
deploy_spec="-f \"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml\""
|
||||
cleanup_spec="-f \"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml\""
|
||||
fi
|
||||
|
||||
kubectl delete ${deploy_spec}
|
||||
helm uninstall --namespace=kube-system kata-deploy --wait
|
||||
kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
|
||||
|
||||
# Let the `kata-deploy` script take care of the runtime class creation / removal
|
||||
yq -i \
|
||||
'.spec.template.spec.containers[0].env[4].value = "true"' \
|
||||
"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
|
||||
# Create the runtime class only for the shim that's being tested
|
||||
yq -i \
|
||||
".spec.template.spec.containers[0].env[2].value = \"${KATA_HYPERVISOR}\"" \
|
||||
"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
|
||||
# Set the tested hypervisor as the default `kata` shim
|
||||
yq -i \
|
||||
".spec.template.spec.containers[0].env[3].value = \"${KATA_HYPERVISOR}\"" \
|
||||
"tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
|
||||
# Let the `kata-deploy` create the default `kata` runtime class
|
||||
yq -i \
|
||||
'.spec.template.spec.containers[0].env[5].value = "true"' \
|
||||
"tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
|
||||
|
||||
sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
|
||||
cat "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
|
||||
grep "${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}" "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" || die "Failed to setup the tests image"
|
||||
|
||||
kubectl apply ${cleanup_spec}
|
||||
sleep 30s
|
||||
|
||||
kubectl delete ${cleanup_spec}
|
||||
kubectl delete -f "tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
|
||||
|
||||
popd
|
||||
}
|
||||
|
||||
@ -7,6 +7,7 @@
|
||||
tests_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "${tests_dir}/common.bash"
|
||||
kubernetes_dir="${tests_dir}/integration/kubernetes"
|
||||
helm_chart_dir="${repo_root_dir}/tools/packaging/kata-deploy/helm-chart/kata-deploy"
|
||||
|
||||
AZ_APPID="${AZ_APPID:-}"
|
||||
AZ_PASSWORD="${AZ_PASSWORD:-}"
|
||||
@ -14,6 +15,22 @@ AZ_SUBSCRIPTION_ID="${AZ_SUBSCRIPTION_ID:-}"
|
||||
AZ_TENANT_ID="${AZ_TENANT_ID:-}"
|
||||
GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-oci-distribution}"
|
||||
GH_PR_NUMBER="${GH_PR_NUMBER:-}"
|
||||
HELM_DEFAULT_INSTALLATION="${HELM_DEFAULT_INSTALLATION:-false}"
|
||||
HELM_AGENT_HTTPS_PROXY="${HELM_AGENT_HTTPS_PROXY:-}"
|
||||
HELM_AGENT_NO_PROXY="${HELM_AGENT_NO_PROXY:-}"
|
||||
HELM_ALLOWED_HYPERVISOR_ANNOTATIONS="${HELM_ALLOWED_HYPERVISOR_ANNOTATIONS:-}"
|
||||
HELM_CREATE_RUNTIME_CLASSES="${HELM_CREATE_RUNTIME_CLASSES:-}"
|
||||
HELM_CREATE_DEFAULT_RUNTIME_CLASS="${HELM_CREATE_DEFAULT_RUNTIME_CLASS:-}"
|
||||
HELM_DEBUG="${HELM_DEBUG:-}"
|
||||
HELM_DEFAULT_SHIM="${HELM_DEFAULT_SHIM:-}"
|
||||
HELM_HOST_OS="${HELM_HOST_OS:-}"
|
||||
HELM_IMAGE_REFERENCE="${HELM_IMAGE_REFERENCE:-}"
|
||||
HELM_IMAGE_TAG="${HELM_IMAGE_TAG:-}"
|
||||
HELM_K8S_DISTRIBUTION="${HELM_K8S_DISTRIBUTION:-}"
|
||||
HELM_PULL_TYPE_MAPPING="${HELM_PULL_TYPE_MAPPING:-}"
|
||||
HELM_SHIMS="${HELM_SHIMS:-}"
|
||||
HELM_SNAPSHOTTER_HANDLER_MAPPING="${HELM_SNAPSHOTTER_HANDLER_MAPPING:-}"
|
||||
KATA_DEPLOY_WAIT_TIMEOUT="${KATA_DEPLOY_WAIT_TIMEOUT:-600}"
|
||||
KATA_HOST_OS="${KATA_HOST_OS:-}"
|
||||
KUBERNETES="${KUBERNETES:-}"
|
||||
K8S_TEST_HOST_TYPE="${K8S_TEST_HOST_TYPE:-small}"
|
||||
@ -427,3 +444,95 @@ function delete_test_runners(){
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function helm_helper() {
|
||||
local max_tries
|
||||
local interval
|
||||
local i
|
||||
local values_yaml
|
||||
|
||||
ensure_yq
|
||||
ensure_helm
|
||||
|
||||
values_yaml=$(mktemp -t values_yaml.XXXXXX)
|
||||
|
||||
if [[ -z "${HELM_IMAGE_REFERENCE}" ]]; then
|
||||
die "HELM_IMAGE_REFERENCE environment variable cannot be empty."
|
||||
fi
|
||||
yq -i ".image.reference = \"${HELM_IMAGE_REFERENCE}\"" "${values_yaml}"
|
||||
|
||||
if [[ -z "${HELM_IMAGE_TAG}" ]]; then
|
||||
die "HELM_IMAGE_TAG environment variable cannot be empty."
|
||||
fi
|
||||
yq -i ".image.tag = \"${HELM_IMAGE_TAG}\"" "${values_yaml}"
|
||||
|
||||
[[ -n "${HELM_K8S_DISTRIBUTION}" ]] && yq -i ".k8sDistribution = \"${HELM_K8S_DISTRIBUTION}\"" "${values_yaml}"
|
||||
|
||||
if [[ "${HELM_DEFAULT_INSTALLATION}" = "false" ]]; then
|
||||
[[ -n "${HELM_DEBUG}" ]] && yq -i ".env.debug = \"${HELM_DEBUG}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_SHIMS}" ]] && yq -i ".env.shims = \"${HELM_SHIMS}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_DEFAULT_SHIM}" ]] && yq -i ".env.defaultShim = \"${HELM_DEFAULT_SHIM}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_CREATE_RUNTIME_CLASSES}" ]] && yq -i ".env.createRuntimeClasses = \"${HELM_CREATE_RUNTIME_CLASSES}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_CREATE_DEFAULT_RUNTIME_CLASS}" ]] && yq -i ".env.createDefaultRuntimeClass = \"${HELM_CREATE_DEFAULT_RUNTIME_CLASS}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_ALLOWED_HYPERVISOR_ANNOTATIONS}" ]] && yq -i ".env.allowedHypervisorAnnotations = \"${HELM_ALLOWED_HYPERVISOR_ANNOTATIONS}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_SNAPSHOTTER_HANDLER_MAPPING}" ]] && yq -i ".env.snapshotterHandlerMapping = \"${HELM_SNAPSHOTTER_HANDLER_MAPPING}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_AGENT_HTTPS_PROXY}" ]] && yq -i ".env.agentHttpsProxy = \"${HELM_AGENT_HTTPS_PROXY}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_AGENT_NO_PROXY}" ]] && yq -i ".env.agentNoProxy = \"${HELM_AGENT_NO_PROXY}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_PULL_TYPE_MAPPING}" ]] && yq -i ".env.pullTypeMapping = \"${HELM_PULL_TYPE_MAPPING}\"" "${values_yaml}"
|
||||
[[ -n "${HELM_HOST_OS}" ]] && yq -i ".env.hostOS=\"${HELM_HOST_OS}\"" "${values_yaml}"
|
||||
fi
|
||||
|
||||
echo "::group::Final kata-deploy manifests used in the test"
|
||||
cat "${values_yaml}"
|
||||
echo ""
|
||||
helm template "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system
|
||||
[[ "$(yq .image.reference "${values_yaml}")" = "${HELM_IMAGE_REFERENCE}" ]] || die "Failed to set image reference"
|
||||
[[ "$(yq .image.tag "${values_yaml}")" = "${HELM_IMAGE_TAG}" ]] || die "Failed to set image tag"
|
||||
echo "::endgroup::"
|
||||
|
||||
max_tries=3
|
||||
interval=10
|
||||
i=10
|
||||
|
||||
# Retry loop for helm install to prevent transient failures due to instantly unreachable cluster
|
||||
set +e # Disable immediate exit on failure
|
||||
while true; do
|
||||
helm upgrade --install kata-deploy "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system --debug
|
||||
ret=${?}
|
||||
if [[ ${ret} -eq 0 ]]; then
|
||||
echo "Helm install succeeded!"
|
||||
break
|
||||
fi
|
||||
i=$((i+1))
|
||||
if [[ ${i} -lt ${max_tries} ]]; then
|
||||
echo "Retrying after ${interval} seconds (Attempt ${i} of $((max_tries - 1)))"
|
||||
else
|
||||
break
|
||||
fi
|
||||
sleep "${interval}"
|
||||
done
|
||||
set -e # Re-enable immediate exit on failure
|
||||
if [[ ${i} -eq ${max_tries} ]]; then
|
||||
die "Failed to deploy kata-deploy after ${max_tries} tries"
|
||||
fi
|
||||
|
||||
# `helm install --wait` does not take effect on single replicas and maxUnavailable=1 DaemonSets
|
||||
# like kata-deploy on CI. So wait for pods being Running in the "traditional" way.
|
||||
local cmd
|
||||
cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\<Running\>'"
|
||||
waitForProcess "${KATA_DEPLOY_WAIT_TIMEOUT}" 10 "${cmd}"
|
||||
|
||||
# FIXME: This is needed as the kata-deploy pod will be set to "Ready"
|
||||
# when it starts running, which may cause issues like not having the
|
||||
# node properly labeled or the artefacts properly deployed when the
|
||||
# tests actually start running.
|
||||
sleep 60s
|
||||
|
||||
echo "::group::kata-deploy logs"
|
||||
kubectl_retry -n kube-system logs --tail=100 -l name=kata-deploy
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::Runtime classes"
|
||||
kubectl_retry get runtimeclass
|
||||
echo "::endgroup::"
|
||||
}
|
||||
|
||||
@ -17,24 +17,22 @@ source "${kubernetes_dir}/../../gha-run-k8s-common.sh"
|
||||
source "${kubernetes_dir}/confidential_kbs.sh"
|
||||
# shellcheck disable=2154
|
||||
tools_dir="${repo_root_dir}/tools"
|
||||
helm_chart_dir="${tools_dir}/packaging/kata-deploy/helm-chart/kata-deploy"
|
||||
kata_tarball_dir="${2:-kata-artifacts}"
|
||||
|
||||
DOCKER_REGISTRY=${DOCKER_REGISTRY:-quay.io}
|
||||
DOCKER_REPO=${DOCKER_REPO:-kata-containers/kata-deploy-ci}
|
||||
DOCKER_TAG=${DOCKER_TAG:-kata-containers-latest}
|
||||
KATA_DEPLOY_WAIT_TIMEOUT=${KATA_DEPLOY_WAIT_TIMEOUT:-600}
|
||||
SNAPSHOTTER_DEPLOY_WAIT_TIMEOUT=${SNAPSHOTTER_DEPLOY_WAIT_TIMEOUT:-8m}
|
||||
KATA_HYPERVISOR=${KATA_HYPERVISOR:-qemu}
|
||||
CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-containerd}
|
||||
KBS=${KBS:-false}
|
||||
KBS_INGRESS=${KBS_INGRESS:-}
|
||||
KUBERNETES="${KUBERNETES:-}"
|
||||
SNAPSHOTTER="${SNAPSHOTTER:-}"
|
||||
ITA_KEY="${ITA_KEY:-}"
|
||||
HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy:-}}"
|
||||
NO_PROXY="${NO_PROXY:-${no_proxy:-}}"
|
||||
PULL_TYPE="${PULL_TYPE:-default}"
|
||||
export DOCKER_REGISTRY="${DOCKER_REGISTRY:-quay.io}"
|
||||
export DOCKER_REPO="${DOCKER_REPO:-kata-containers/kata-deploy-ci}"
|
||||
export DOCKER_TAG="${DOCKER_TAG:-kata-containers-latest}"
|
||||
export SNAPSHOTTER_DEPLOY_WAIT_TIMEOUT="${SNAPSHOTTER_DEPLOY_WAIT_TIMEOUT:-8m}"
|
||||
export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
|
||||
export CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-containerd}"
|
||||
export KBS="${KBS:-false}"
|
||||
export KBS_INGRESS="${KBS_INGRESS:-}"
|
||||
export KUBERNETES="${KUBERNETES:-}"
|
||||
export SNAPSHOTTER="${SNAPSHOTTER:-}"
|
||||
export ITA_KEY="${ITA_KEY:-}"
|
||||
export HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy:-}}"
|
||||
export NO_PROXY="${NO_PROXY:-${no_proxy:-}}"
|
||||
export PULL_TYPE="${PULL_TYPE:-default}"
|
||||
export AUTO_GENERATE_POLICY="${AUTO_GENERATE_POLICY:-no}"
|
||||
export TEST_CLUSTER_NAMESPACE="${TEST_CLUSTER_NAMESPACE:-kata-containers-k8s-tests}"
|
||||
export GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-oci-distribution}"
|
||||
@ -165,8 +163,6 @@ function deploy_coco_kbs() {
|
||||
|
||||
function deploy_kata() {
|
||||
platform="${1:-}"
|
||||
ensure_helm
|
||||
ensure_yq
|
||||
|
||||
[[ "${platform}" = "kcli" ]] && \
|
||||
export KUBECONFIG="${HOME}/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig"
|
||||
@ -177,106 +173,44 @@ function deploy_kata() {
|
||||
|
||||
set_default_cluster_namespace
|
||||
|
||||
local values_yaml
|
||||
values_yaml=$(mktemp /tmp/values_yaml.XXXXXX)
|
||||
|
||||
yq -i ".k8sDistribution = \"${KUBERNETES}\"" "${values_yaml}"
|
||||
yq -i ".image.reference = \"${DOCKER_REGISTRY}/${DOCKER_REPO}\"" "${values_yaml}"
|
||||
yq -i ".image.tag = \"${DOCKER_TAG}\"" "${values_yaml}"
|
||||
yq -i ".env.debug = \"true\"" "${values_yaml}"
|
||||
yq -i ".env.shims = \"${KATA_HYPERVISOR}\"" "${values_yaml}"
|
||||
yq -i ".env.defaultShim = \"${KATA_HYPERVISOR}\"" "${values_yaml}"
|
||||
yq -i ".env.createRuntimeClasses = \"true\"" "${values_yaml}"
|
||||
yq -i ".env.createDefaultRuntimeClass = \"true\"" "${values_yaml}"
|
||||
yq -i ".env.allowedHypervisorAnnotations = \"default_vcpus\"" "${values_yaml}"
|
||||
yq -i ".env.snapshotterHandlerMapping = \"\"" "${values_yaml}"
|
||||
yq -i ".env.agentHttpsProxy = \"\"" "${values_yaml}"
|
||||
yq -i ".env.agentNoProxy = \"\"" "${values_yaml}"
|
||||
yq -i ".env.pullTypeMapping = \"\"" "${values_yaml}"
|
||||
yq -i ".env.hostOS = \"\"" "${values_yaml}"
|
||||
|
||||
if [[ -n "${SNAPSHOTTER}" ]]; then
|
||||
yq -i ".env.snapshotterHandlerMapping = \"${KATA_HYPERVISOR}:${SNAPSHOTTER}\"" "${values_yaml}"
|
||||
fi
|
||||
|
||||
ANNOTATIONS="default_vcpus"
|
||||
if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
|
||||
yq -i ".env.allowedHypervisorAnnotations = \"image kernel default_vcpus\"" "${values_yaml}"
|
||||
yq -i ".env.hostOS = \"${KATA_HOST_OS}\"" "${values_yaml}"
|
||||
ANNOTATIONS="image kernel default_vcpus"
|
||||
fi
|
||||
|
||||
if [[ "${KATA_HYPERVISOR}" = "qemu" ]]; then
|
||||
yq -i ".env.allowedHypervisorAnnotations = \"image initrd kernel default_vcpus\"" "${values_yaml}"
|
||||
ANNOTATIONS="image initrd kernel default_vcpus"
|
||||
fi
|
||||
|
||||
if [[ "${KATA_HYPERVISOR}" = "qemu-tdx" ]]; then
|
||||
yq -i ".env.agentHttpsProxy = \"${HTTPS_PROXY}\"" "${values_yaml}"
|
||||
yq -i ".env.agentNoProxy = \"${NO_PROXY}\"" "${values_yaml}"
|
||||
SNAPSHOTTER_HANDLER_MAPPING=""
|
||||
if [[ -n "${SNAPSHOTTER}" ]]; then
|
||||
SNAPSHOTTER_HANDLER_MAPPING="${KATA_HYPERVISOR}:${SNAPSHOTTER}"
|
||||
fi
|
||||
|
||||
# Set the PULL_TYPE_MAPPING
|
||||
PULL_TYPE_MAPPING=""
|
||||
if [[ "${PULL_TYPE}" != "default" ]]; then
|
||||
yq -i ".env.pullTypeMapping = \"${KATA_HYPERVISOR}:${PULL_TYPE}\"" "${values_yaml}"
|
||||
PULL_TYPE_MAPPING="${KATA_HYPERVISOR}:${PULL_TYPE}"
|
||||
fi
|
||||
|
||||
echo "::group::Final kata-deploy manifests used in the test"
|
||||
cat "${values_yaml}"
|
||||
helm template "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system
|
||||
[[ "$(yq .image.reference "${values_yaml}")" = "${DOCKER_REGISTRY}/${DOCKER_REPO}" ]] || die "Failed to set image reference"
|
||||
[[ "$(yq .image.tag "${values_yaml}")" = "${DOCKER_TAG}" ]] || die "Failed to set image tag"
|
||||
echo "::endgroup::"
|
||||
|
||||
local max_tries
|
||||
local interval
|
||||
local i
|
||||
|
||||
max_tries=3
|
||||
interval=10
|
||||
i=10
|
||||
|
||||
# Retry loop for helm install to prevent transient failures due to instantly unreachable cluster
|
||||
set +e # Disable immediate exit on failure
|
||||
while true; do
|
||||
helm upgrade --install kata-deploy "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system --debug
|
||||
ret=${?}
|
||||
if [[ ${ret} -eq 0 ]]; then
|
||||
echo "Helm install succeeded!"
|
||||
break
|
||||
fi
|
||||
i=$((i+1))
|
||||
if [[ ${i} -lt ${max_tries} ]]; then
|
||||
echo "Retrying after ${interval} seconds (Attempt ${i} of $((max_tries - 1)))"
|
||||
else
|
||||
break
|
||||
fi
|
||||
sleep "${interval}"
|
||||
done
|
||||
set -e # Re-enable immediate exit on failure
|
||||
if [[ ${i} -eq ${max_tries} ]]; then
|
||||
die "Failed to deploy kata-deploy after ${max_tries} tries"
|
||||
HOST_OS=""
|
||||
if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
|
||||
HOST_OS="${KATA_HOST_OS}"
|
||||
fi
|
||||
|
||||
# `helm install --wait` does not take effect on single replicas and maxUnavailable=1 DaemonSets
|
||||
# like kata-deploy on CI. So wait for pods being Running in the "tradicional" way.
|
||||
local cmd
|
||||
cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\<Running\>'"
|
||||
waitForProcess "${KATA_DEPLOY_WAIT_TIMEOUT}" 10 "${cmd}"
|
||||
|
||||
# This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
|
||||
# which may cause issues like not having the node properly labeled or the artefacts
|
||||
# properly deployed when the tests actually start running.
|
||||
if [[ "${platform}" = "aks" ]]; then
|
||||
sleep 240s
|
||||
else
|
||||
sleep 60s
|
||||
fi
|
||||
|
||||
echo "::group::kata-deploy logs"
|
||||
kubectl_retry -n kube-system logs --tail=100 -l name=kata-deploy
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::Runtime classes"
|
||||
kubectl_retry get runtimeclass
|
||||
echo "::endgroup::"
|
||||
export HELM_K8S_DISTRIBUTION="${KUBERNETES}"
|
||||
export HELM_IMAGE_REFERENCE="${DOCKER_REGISTRY}/${DOCKER_REPO}"
|
||||
export HELM_IMAGE_TAG="${DOCKER_TAG}"
|
||||
export HELM_DEBUG="true"
|
||||
export HELM_SHIMS="${KATA_HYPERVISOR}"
|
||||
export HELM_DEFAULT_SHIM="${KATA_HYPERVISOR}"
|
||||
export HELM_CREATE_RUNTIME_CLASSES="true"
|
||||
export HELM_CREATE_DEFAULT_RUNTIME_CLASS="true"
|
||||
export HELM_ALLOWED_HYPERVISOR_ANNOTATIONS="${ANNOTATIONS}"
|
||||
export HELM_SNAPSHOTTER_HANDLER_MAPPING="${SNAPSHOTTER_HANDLER_MAPPING}"
|
||||
export HELM_AGENT_HTTPS_PROXY="${HTTPS_PROXY}"
|
||||
export HELM_AGENT_NO_PROXY="${NO_PROXY}"
|
||||
export HELM_PULL_TYPE_MAPPING="${PULL_TYPE_MAPPING}"
|
||||
export HELM_HOST_OS="${HOST_OS}"
|
||||
helm_helper
|
||||
}
|
||||
|
||||
function install_kbs_client() {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user