mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-04-27 03:21:04 +00:00
shellcheck: tests: k8s: Fix gha-run.sh warnings
As we'll touch this file during this series, let's already make sure we solve all the needed warnings. Signed-off-by: Fabiano Fidêncio <fabiano@fidencio.org>
This commit is contained in:
Fabiano Fidêncio
parent
50f765b19c
commit
545780a83a
@ -9,7 +9,7 @@ set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
DEBUG="${DEBUG:-}"
|
||||
[ -n "$DEBUG" ] && set -x
|
||||
[[ -n "${DEBUG}" ]] && set -x
|
||||
|
||||
kubernetes_dir="$(dirname "$(readlink -f "$0")")"
|
||||
source "${kubernetes_dir}/../../gha-run-k8s-common.sh"
|
||||
@ -118,7 +118,7 @@ EOF
|
||||
sleep 60s
|
||||
sudo cat "${containerd_config_file}"
|
||||
|
||||
if [ "${KUBERNETES}" = 'k3s' ]
|
||||
if [[ "${KUBERNETES}" = 'k3s' ]]
|
||||
then
|
||||
local ctr_dm_status
|
||||
local result
|
||||
@ -128,9 +128,9 @@ EOF
|
||||
plugins ls |\
|
||||
awk '$2 ~ /^devmapper$/ { print $0 }' || true)
|
||||
|
||||
result=$(echo "$ctr_dm_status" | awk '{print $4}' || true)
|
||||
result=$(echo "${ctr_dm_status}" | awk '{print $4}' || true)
|
||||
|
||||
[ "$result" = 'ok' ] || die "k3s containerd device mapper not configured: '$ctr_dm_status'"
|
||||
[[ "${result}" = 'ok' ]] || die "k3s containerd device mapper not configured: '${ctr_dm_status}'"
|
||||
fi
|
||||
|
||||
info "devicemapper (DM) devices"
|
||||
@ -160,7 +160,7 @@ function delete_coco_kbs() {
|
||||
# service externally
|
||||
#
|
||||
function deploy_coco_kbs() {
|
||||
kbs_k8s_deploy "$KBS_INGRESS"
|
||||
kbs_k8s_deploy "${KBS_INGRESS}"
|
||||
}
|
||||
|
||||
function deploy_kata() {
|
||||
@ -168,10 +168,10 @@ function deploy_kata() {
|
||||
ensure_helm
|
||||
ensure_yq
|
||||
|
||||
[ "$platform" = "kcli" ] && \
|
||||
export KUBECONFIG="$HOME/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig"
|
||||
[[ "${platform}" = "kcli" ]] && \
|
||||
export KUBECONFIG="${HOME}/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig"
|
||||
|
||||
if [ "${K8S_TEST_HOST_TYPE}" = "baremetal" ]; then
|
||||
if [[ "${K8S_TEST_HOST_TYPE}" = "baremetal" ]]; then
|
||||
cleanup_kata_deploy || true
|
||||
fi
|
||||
|
||||
@ -195,65 +195,76 @@ function deploy_kata() {
|
||||
yq -i ".env.pullTypeMapping = \"\"" "${values_yaml}"
|
||||
yq -i ".env.hostOS = \"\"" "${values_yaml}"
|
||||
|
||||
if [ -n "${SNAPSHOTTER}" ]; then
|
||||
if [[ -n "${SNAPSHOTTER}" ]]; then
|
||||
yq -i ".env.snapshotterHandlerMapping = \"${KATA_HYPERVISOR}:${SNAPSHOTTER}\"" "${values_yaml}"
|
||||
fi
|
||||
|
||||
if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
|
||||
if [[ "${KATA_HOST_OS}" = "cbl-mariner" ]]; then
|
||||
yq -i ".env.allowedHypervisorAnnotations = \"image kernel default_vcpus\"" "${values_yaml}"
|
||||
yq -i ".env.hostOS = \"${KATA_HOST_OS}\"" "${values_yaml}"
|
||||
fi
|
||||
|
||||
if [ "${KATA_HYPERVISOR}" = "qemu" ]; then
|
||||
if [[ "${KATA_HYPERVISOR}" = "qemu" ]]; then
|
||||
yq -i ".env.allowedHypervisorAnnotations = \"image initrd kernel default_vcpus\"" "${values_yaml}"
|
||||
fi
|
||||
|
||||
if [ "${KATA_HYPERVISOR}" = "qemu-tdx" ]; then
|
||||
if [[ "${KATA_HYPERVISOR}" = "qemu-tdx" ]]; then
|
||||
yq -i ".env.agentHttpsProxy = \"${HTTPS_PROXY}\"" "${values_yaml}"
|
||||
yq -i ".env.agentNoProxy = \"${NO_PROXY}\"" "${values_yaml}"
|
||||
fi
|
||||
|
||||
# Set the PULL_TYPE_MAPPING
|
||||
if [ "${PULL_TYPE}" != "default" ]; then
|
||||
if [[ "${PULL_TYPE}" != "default" ]]; then
|
||||
yq -i ".env.pullTypeMapping = \"${KATA_HYPERVISOR}:${PULL_TYPE}\"" "${values_yaml}"
|
||||
fi
|
||||
|
||||
echo "::group::Final kata-deploy manifests used in the test"
|
||||
cat "${values_yaml}"
|
||||
helm template "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system
|
||||
[ "$(yq .image.reference ${values_yaml})" = "${DOCKER_REGISTRY}/${DOCKER_REPO}" ] || die "Failed to set image reference"
|
||||
[ "$(yq .image.tag ${values_yaml})" = "${DOCKER_TAG}" ] || die "Failed to set image tag"
|
||||
[[ "$(yq .image.reference "${values_yaml}")" = "${DOCKER_REGISTRY}/${DOCKER_REPO}" ]] || die "Failed to set image reference"
|
||||
[[ "$(yq .image.tag "${values_yaml}")" = "${DOCKER_TAG}" ]] || die "Failed to set image tag"
|
||||
echo "::endgroup::"
|
||||
|
||||
local max_tries=3
|
||||
local interval=10
|
||||
local i=0
|
||||
local max_tries
|
||||
local interval
|
||||
local i
|
||||
|
||||
max_tries=3
|
||||
interval=10
|
||||
i=10
|
||||
|
||||
# Retry loop for helm install to prevent transient failures due to instantly unreachable cluster
|
||||
set +e # Disable immediate exit on failure
|
||||
while true; do
|
||||
helm upgrade --install kata-deploy "${helm_chart_dir}" --values "${values_yaml}" --namespace kube-system --debug
|
||||
if [ $? -eq 0 ]; then
|
||||
ret=${?}
|
||||
if [[ ${ret} -eq 0 ]]; then
|
||||
echo "Helm install succeeded!"
|
||||
break
|
||||
fi
|
||||
i=$((i+1))
|
||||
[ $i -lt $max_tries ] && echo "Retrying after $interval seconds (Attempt $i of $(($max_tries - 1)))" || break
|
||||
sleep $interval
|
||||
if [[ ${i} -lt ${max_tries} ]]; then
|
||||
echo "Retrying after ${interval} seconds (Attempt ${i} of $((max_tries - 1)))"
|
||||
else
|
||||
break
|
||||
fi
|
||||
sleep "${interval}"
|
||||
done
|
||||
set -e # Re-enable immediate exit on failure
|
||||
if [ $i -eq $max_tries ]; then
|
||||
die "Failed to deploy kata-deploy after $max_tries tries"
|
||||
if [[ ${i} -eq ${max_tries} ]]; then
|
||||
die "Failed to deploy kata-deploy after ${max_tries} tries"
|
||||
fi
|
||||
|
||||
# `helm install --wait` does not take effect on single replicas and maxUnavailable=1 DaemonSets
|
||||
# like kata-deploy on CI. So wait for pods being Running in the "tradicional" way.
|
||||
local cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\<Running\>'"
|
||||
waitForProcess "${KATA_DEPLOY_WAIT_TIMEOUT}" 10 "$cmd"
|
||||
local cmd
|
||||
cmd="kubectl -n kube-system get -l name=kata-deploy pod 2>/dev/null | grep '\<Running\>'"
|
||||
waitForProcess "${KATA_DEPLOY_WAIT_TIMEOUT}" 10 "${cmd}"
|
||||
|
||||
# This is needed as the kata-deploy pod will be set to "Ready" when it starts running,
|
||||
# which may cause issues like not having the node properly labeled or the artefacts
|
||||
# properly deployed when the tests actually start running.
|
||||
if [ "${platform}" = "aks" ]; then
|
||||
if [[ "${platform}" = "aks" ]]; then
|
||||
sleep 240s
|
||||
else
|
||||
sleep 60s
|
||||
@ -277,7 +288,7 @@ function uninstall_kbs_client() {
|
||||
}
|
||||
|
||||
function run_tests() {
|
||||
if [ "${K8S_TEST_HOST_TYPE}" = "baremetal" ]; then
|
||||
if [[ "${K8S_TEST_HOST_TYPE}" = "baremetal" ]]; then
|
||||
# Baremetal self-hosted runners end up accumulating way too much log
|
||||
# and when those get displayed it's very hard to understand what's
|
||||
# part of the current run and what's something from the past coming
|
||||
@ -291,10 +302,10 @@ function run_tests() {
|
||||
ensure_yq
|
||||
platform="${1:-}"
|
||||
|
||||
[ "$platform" = "kcli" ] && \
|
||||
export KUBECONFIG="$HOME/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig"
|
||||
[[ "${platform}" = "kcli" ]] && \
|
||||
export KUBECONFIG="${HOME}/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig"
|
||||
|
||||
if [ "${AUTO_GENERATE_POLICY}" = "yes" ] && [ "${GENPOLICY_PULL_METHOD}" = "containerd" ]; then
|
||||
if [[ "${AUTO_GENERATE_POLICY}" = "yes" ]] && [[ "${GENPOLICY_PULL_METHOD}" = "containerd" ]]; then
|
||||
# containerd's config on the local machine (where kubectl and genpolicy are executed by CI),
|
||||
# might have been provided by a distro-specific package that disables the cri plug-in by using:
|
||||
#
|
||||
@ -309,10 +320,15 @@ function run_tests() {
|
||||
sudo systemctl restart containerd && sudo systemctl is-active containerd
|
||||
|
||||
# Allow genpolicy to access the containerd image pull APIs without sudo.
|
||||
local socket_wait_time=30
|
||||
local socket_sleep_time=3
|
||||
local cmd="sudo chmod a+rw /var/run/containerd/containerd.sock"
|
||||
waitForProcess "${socket_wait_time}" "${socket_sleep_time}" "$cmd"
|
||||
local socket_wait_time
|
||||
local socket_sleep_time
|
||||
local cmd
|
||||
|
||||
socket_wait_time=30
|
||||
socket_sleep_time=3
|
||||
cmd="sudo chmod a+rw /var/run/containerd/containerd.sock"
|
||||
|
||||
waitForProcess "${socket_wait_time}" "${socket_sleep_time}" "${cmd}"
|
||||
fi
|
||||
|
||||
set_test_cluster_namespace
|
||||
@ -323,22 +339,22 @@ function run_tests() {
|
||||
# In case of running on Github workflow it needs to save the start time
|
||||
# on the environment variables file so that the variable is exported on
|
||||
# next workflow steps.
|
||||
if [ -n "${GITHUB_ENV:-}" ]; then
|
||||
if [[ -n "${GITHUB_ENV:-}" ]]; then
|
||||
start_time=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
export start_time
|
||||
echo "start_time=${start_time}" >> "$GITHUB_ENV"
|
||||
echo "start_time=${start_time}" >> "${GITHUB_ENV}"
|
||||
fi
|
||||
|
||||
if [[ "${KATA_HYPERVISOR}" = "cloud-hypervisor" ]] && [[ "${SNAPSHOTTER}" = "devmapper" ]]; then
|
||||
if [ -n "$GITHUB_ENV" ]; then
|
||||
if [[ -n "${GITHUB_ENV}" ]]; then
|
||||
KATA_TEST_VERBOSE=true
|
||||
export KATA_TEST_VERBOSE
|
||||
echo "KATA_TEST_VERBOSE=${KATA_TEST_VERBOSE}" >> "$GITHUB_ENV"
|
||||
echo "KATA_TEST_VERBOSE=${KATA_TEST_VERBOSE}" >> "${GITHUB_ENV}"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${KATA_HYPERVISOR}" = "dragonball" ]] && [[ "${SNAPSHOTTER}" = "devmapper" ]]; then
|
||||
echo "Skipping tests for $KATA_HYPERVISOR using devmapper"
|
||||
echo "Skipping tests for ${KATA_HYPERVISOR} using devmapper"
|
||||
else
|
||||
bash run_kubernetes_tests.sh
|
||||
fi
|
||||
@ -346,24 +362,32 @@ function run_tests() {
|
||||
}
|
||||
|
||||
function collect_artifacts() {
|
||||
if [ -z "${start_time:-}" ]; then
|
||||
if [[ -z "${start_time:-}" ]]; then
|
||||
warn "tests start time is not defined. Cannot gather journal information"
|
||||
return
|
||||
fi
|
||||
|
||||
local artifacts_dir="/tmp/artifacts"
|
||||
if [ -d "${artifacts_dir}" ]; then
|
||||
local artifacts_dir
|
||||
artifacts_dir="/tmp/artifacts"
|
||||
if [[ -d "${artifacts_dir}" ]]; then
|
||||
rm -rf "${artifacts_dir}"
|
||||
fi
|
||||
mkdir -p "${artifacts_dir}"
|
||||
info "Collecting artifacts using ${KATA_HYPERVISOR} hypervisor"
|
||||
local journalctl_log_filename="journalctl-$RANDOM.log"
|
||||
local journalctl_log_path="${artifacts_dir}/${journalctl_log_filename}"
|
||||
sudo journalctl --since="$start_time" > "${journalctl_log_path}"
|
||||
local journalctl_log_filename
|
||||
local journalctl_log_path
|
||||
|
||||
local k3s_dir='/var/lib/rancher/k3s/agent'
|
||||
journalctl_log_filename="journalctl-${RANDOM}.log"
|
||||
journalctl_log_path="${artifacts_dir}/${journalctl_log_filename}"
|
||||
|
||||
if [ -d "$k3s_dir" ]
|
||||
# As we want to call journalctl with sudo, we're safe to ignore SC2024 here
|
||||
# shellcheck disable=SC2024
|
||||
sudo journalctl --since="${start_time}" > "${journalctl_log_path}"
|
||||
|
||||
local k3s_dir
|
||||
k3s_dir='/var/lib/rancher/k3s/agent'
|
||||
|
||||
if [[ -d "${k3s_dir}" ]]
|
||||
then
|
||||
info "Collecting k3s artifacts"
|
||||
|
||||
@ -375,36 +399,35 @@ function collect_artifacts() {
|
||||
files+=('containerd/containerd.log')
|
||||
|
||||
# Add any rotated containerd logs
|
||||
files+=( $(sudo find \
|
||||
"${k3s_dir}/containerd/" \
|
||||
-type f \
|
||||
-name 'containerd*\.log\.gz') )
|
||||
files+=("$(sudo find "${k3s_dir}/containerd/" -type f -name 'containerd*\.log\.gz')")
|
||||
|
||||
local file
|
||||
|
||||
for file in "${files[@]}"
|
||||
do
|
||||
local path="$k3s_dir/$file"
|
||||
sudo [ ! -e "$path" ] && continue
|
||||
local path="${k3s_dir}/${file}"
|
||||
sudo [[ ! -e "${path}" ]] && continue
|
||||
|
||||
local encoded
|
||||
encoded=$(echo "$path" | tr '/' '-' | sed 's/^-//g')
|
||||
|
||||
local from="$path"
|
||||
encoded="$(echo "${path}" | tr '/' '-' | sed 's/^-//g')"
|
||||
|
||||
local from
|
||||
local to
|
||||
|
||||
from="${path}"
|
||||
to="${artifacts_dir}/${encoded}"
|
||||
|
||||
if [[ $path = *.gz ]]
|
||||
if [[ ${path} = *.gz ]]
|
||||
then
|
||||
sudo cp "$from" "$to"
|
||||
sudo cp "${from}" "${to}"
|
||||
else
|
||||
to="${to}.gz"
|
||||
sudo gzip -c "$from" > "$to"
|
||||
# As we want to call gzip with sudo, we're safe to ignore SC2024 here
|
||||
# shellcheck disable=SC2024
|
||||
sudo gzip -c "${from}" > "${to}"
|
||||
fi
|
||||
|
||||
info " Collected k3s file '$from' to '$to'"
|
||||
info " Collected k3s file '${from}' to '${to}'"
|
||||
done
|
||||
fi
|
||||
}
|
||||
@ -422,13 +445,13 @@ function cleanup() {
|
||||
test_type="${2:-k8s}"
|
||||
ensure_yq
|
||||
|
||||
[ "$platform" = "kcli" ] && \
|
||||
export KUBECONFIG="$HOME/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig"
|
||||
[[ "${platform}" = "kcli" ]] && \
|
||||
export KUBECONFIG="${HOME}/.kcli/clusters/${CLUSTER_NAME:-kata-k8s}/auth/kubeconfig"
|
||||
|
||||
echo "Gather information about the nodes and pods before cleaning up the node"
|
||||
get_nodes_and_pods_info
|
||||
|
||||
if [ "${platform}" = "aks" ]; then
|
||||
if [[ "${platform}" = "aks" ]]; then
|
||||
delete_cluster "${test_type}"
|
||||
return
|
||||
fi
|
||||
@ -436,7 +459,7 @@ function cleanup() {
|
||||
# In case of canceling workflow manually, 'run_kubernetes_tests.sh' continues running and triggers new tests,
|
||||
# resulting in the CI being in an unexpected state. So we need kill all running test scripts before cleaning up the node.
|
||||
# See issue https://github.com/kata-containers/kata-containers/issues/9980
|
||||
delete_test_runners || true
|
||||
delete_test_runners || true
|
||||
# Switch back to the default namespace and delete the tests one
|
||||
delete_test_cluster_namespace || true
|
||||
|
||||
@ -475,8 +498,9 @@ function deploy_nydus_snapshotter() {
|
||||
echo "::group::deploy_nydus_snapshotter"
|
||||
ensure_yq
|
||||
|
||||
local nydus_snapshotter_install_dir="/tmp/nydus-snapshotter"
|
||||
if [ -d "${nydus_snapshotter_install_dir}" ]; then
|
||||
local nydus_snapshotter_install_dir
|
||||
nydus_snapshotter_install_dir="/tmp/nydus-snapshotter"
|
||||
if [[ -d "${nydus_snapshotter_install_dir}" ]]; then
|
||||
rm -rf "${nydus_snapshotter_install_dir}"
|
||||
fi
|
||||
mkdir -p "${nydus_snapshotter_install_dir}"
|
||||
@ -484,11 +508,11 @@ function deploy_nydus_snapshotter() {
|
||||
nydus_snapshotter_version=$(get_from_kata_deps ".externals.nydus-snapshotter.version")
|
||||
git clone -b "${nydus_snapshotter_version}" "${nydus_snapshotter_url}" "${nydus_snapshotter_install_dir}"
|
||||
|
||||
pushd "$nydus_snapshotter_install_dir"
|
||||
if [ "${K8S_TEST_HOST_TYPE}" = "baremetal" ]; then
|
||||
pushd "${nydus_snapshotter_install_dir}"
|
||||
if [[ "${K8S_TEST_HOST_TYPE}" = "baremetal" ]]; then
|
||||
cleanup_nydus_snapshotter || true
|
||||
fi
|
||||
if [ "${PULL_TYPE}" == "guest-pull" ]; then
|
||||
if [[ "${PULL_TYPE}" == "guest-pull" ]]; then
|
||||
# Enable guest pull feature in nydus snapshotter
|
||||
yq -i \
|
||||
'select(.kind == "ConfigMap").data.FS_DRIVER = "proxy"' \
|
||||
@ -518,14 +542,14 @@ function deploy_nydus_snapshotter() {
|
||||
|
||||
# Deploy nydus snapshotter as a daemonset
|
||||
kubectl_retry create -f "misc/snapshotter/nydus-snapshotter-rbac.yaml"
|
||||
if [ "${KUBERNETES}" = "k3s" ]; then
|
||||
if [[ "${KUBERNETES}" = "k3s" ]]; then
|
||||
kubectl_retry apply -k "misc/snapshotter/overlays/k3s"
|
||||
else
|
||||
kubectl_retry apply -f "misc/snapshotter/base/nydus-snapshotter.yaml"
|
||||
fi
|
||||
popd
|
||||
|
||||
kubectl rollout status daemonset nydus-snapshotter -n nydus-system --timeout ${SNAPSHOTTER_DEPLOY_WAIT_TIMEOUT}
|
||||
kubectl rollout status daemonset nydus-snapshotter -n nydus-system --timeout "${SNAPSHOTTER_DEPLOY_WAIT_TIMEOUT}"
|
||||
|
||||
echo "::endgroup::"
|
||||
echo "::group::nydus snapshotter logs"
|
||||
@ -538,15 +562,16 @@ function deploy_nydus_snapshotter() {
|
||||
|
||||
function cleanup_nydus_snapshotter() {
|
||||
echo "cleanup_nydus_snapshotter"
|
||||
local nydus_snapshotter_install_dir="/tmp/nydus-snapshotter"
|
||||
if [ ! -d "${nydus_snapshotter_install_dir}" ]; then
|
||||
local nydus_snapshotter_install_dir
|
||||
nydus_snapshotter_install_dir="/tmp/nydus-snapshotter"
|
||||
if [[ ! -d "${nydus_snapshotter_install_dir}" ]]; then
|
||||
>&2 echo "nydus snapshotter dir not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
pushd "$nydus_snapshotter_install_dir"
|
||||
pushd "${nydus_snapshotter_install_dir}"
|
||||
|
||||
if [ "${KUBERNETES}" = "k3s" ]; then
|
||||
if [[ "${KUBERNETES}" = "k3s" ]]; then
|
||||
kubectl_retry delete --ignore-not-found -k "misc/snapshotter/overlays/k3s"
|
||||
else
|
||||
kubectl_retry delete --ignore-not-found -f "misc/snapshotter/base/nydus-snapshotter.yaml"
|
||||
@ -567,7 +592,7 @@ function main() {
|
||||
case "${action}" in
|
||||
install-azure-cli) install_azure_cli ;;
|
||||
login-azure) login_azure ;;
|
||||
create-cluster) create_cluster ;;
|
||||
create-cluster) create_cluster "" ;;
|
||||
create-cluster-kcli) create_cluster_kcli ;;
|
||||
configure-snapshotter) configure_snapshotter ;;
|
||||
setup-crio) setup_crio ;;
|
||||
@ -577,7 +602,7 @@ function main() {
|
||||
install-kata-tools) install_kata_tools ;;
|
||||
install-kbs-client) install_kbs_client ;;
|
||||
install-kubectl) install_kubectl ;;
|
||||
get-cluster-credentials) get_cluster_credentials ;;
|
||||
get-cluster-credentials) get_cluster_credentials "" ;;
|
||||
deploy-csi-driver) return 0 ;;
|
||||
deploy-kata) deploy_kata ;;
|
||||
deploy-kata-aks) deploy_kata "aks" ;;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user