Address code review: improve yq parsing and generalize URL handling

- Use yq with .[] to get clean URL output without YAML formatting
- Generalize URL lookup in download-with-oras-cache.sh to try urls array
  first, then fallback to single url field (extensible for other components)

Co-authored-by: sprt <110295+sprt@users.noreply.github.com>

ci/install_libseccomp.sh

@@ -41,12 +41,9 @@ gperf_version="${GPERF_VERSION:-""}"
 if [[ -z "${gperf_version}" ]]; then
 	gperf_version=$(get_from_kata_deps ".externals.gperf.version")
 fi
-gperf_url="${GPERF_URL:-""}"
-if [[ -z "${gperf_url}" ]]; then
-	gperf_url=$(get_from_kata_deps ".externals.gperf.url")
-fi
 gperf_tarball="gperf-${gperf_version}.tar.gz"
-gperf_tarball_url="${gperf_url}/${gperf_tarball}"
+# Path to the urls array in versions.yaml (used by download_from_mirror_list)
+gperf_urls_path=".externals.gperf.urls"
 
 # Use ORAS cache for gperf downloads (gperf upstream can be unreliable)
 USE_ORAS_CACHE="${USE_ORAS_CACHE:-yes}"
@@ -76,6 +73,8 @@ build_and_install_gperf() {
 	echo "Build and install gperf version ${gperf_version}"
 	mkdir -p "${gperf_install_dir}"
 
+	local downloaded_tarball=""
+
 	# Use ORAS cache if available and enabled
 	if [[ "${USE_ORAS_CACHE}" == "yes" ]] && [[ -f "${oras_cache_helper}" ]]; then
 		echo "Using ORAS cache for gperf download"
@@ -83,16 +82,21 @@ build_and_install_gperf() {
 		local cached_tarball
 		cached_tarball=$(download_component gperf "$(pwd)")
 		if [[ -f "${cached_tarball}" ]]; then
-			gperf_tarball="${cached_tarball}"
+			downloaded_tarball="${cached_tarball}"
 		else
-			echo "ORAS cache download failed, falling back to direct download"
-			curl -sLO "${gperf_tarball_url}"
+			echo "ORAS cache download failed, falling back to mirror list"
 		fi
-	else
-		curl -sLO "${gperf_tarball_url}"
 	fi
 
-	tar -xf "${gperf_tarball}"
+	# If ORAS cache failed or was not used, try downloading from mirror list
+	if [[ -z "${downloaded_tarball}" ]]; then
+		downloaded_tarball=$(download_from_mirror_list "${gperf_urls_path}" "${gperf_tarball}" "$(pwd)")
+		if [[ ! -f "${downloaded_tarball}" ]]; then
+			die "Failed to download gperf tarball from any mirror"
+		fi
+	fi
+
+	tar -xf "${downloaded_tarball}"
 	pushd "gperf-${gperf_version}"
 	# Unset $CC for configure, we will always use native for gperf
 	CC="" ./configure --prefix="${gperf_install_dir}"

tests/common.bash

@@ -602,6 +602,60 @@ function get_from_kata_deps() {
         echo "$result"
 }
 
+# Download a file by trying multiple mirror URLs until one succeeds.
+# This function is useful for downloading files from unreliable sources
+# by providing fallback mirrors.
+#
+# $1 - path in versions.yaml to the urls array (e.g., ".externals.gperf.urls")
+# $2 - filename to download (will be appended to each mirror URL)
+# $3 - destination directory (defaults to current directory)
+#
+# Returns: 0 on success, 1 on failure
+# Output: Prints the path to the downloaded file on success
+function download_from_mirror_list() {
+        local urls_path="${1}"
+        local filename="${2}"
+        local dest_dir="${3:-.}"
+        local versions_file="${repo_root_dir}/versions.yaml"
+
+        command -v yq &>/dev/null || die 'yq command is not in your $PATH'
+
+        local urls
+        # Query yq to get URLs as clean lines (using .[] to iterate array elements)
+        local yq_version
+        yq_version=$(yq --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | cut -d. -f1)
+        if [ "$yq_version" -eq 3 ]; then
+                local dependency
+                dependency=$(echo "${urls_path}" | sed "s/^\.//g")
+                urls=$("yq" read -p p "$versions_file" "${dependency}.*")
+        else
+                urls=$("yq" "${urls_path} | .[]" "$versions_file")
+        fi
+
+        if [[ -z "${urls}" ]]; then
+                echo "Error: No URLs found at ${urls_path}" >&2
+                return 1
+        fi
+
+        # Iterate over each URL (one per line)
+        local url
+        while IFS= read -r url; do
+                # Skip empty lines
+                [[ -z "${url}" ]] && continue
+                local full_url="${url}${filename}"
+                echo "Trying to download from: ${full_url}" >&2
+                if curl -sfLo "${dest_dir}/${filename}" "${full_url}"; then
+                        echo "Successfully downloaded ${filename}" >&2
+                        echo "${dest_dir}/${filename}"
+                        return 0
+                fi
+                echo "Failed to download from ${full_url}, trying next mirror..." >&2
+        done <<< "${urls}"
+
+        echo "Error: Failed to download ${filename} from all mirrors" >&2
+        return 1
+}
+
 # project: org/repo format
 # base_version: ${major}.${minor}
 # allow_unstable: Whether alpha / beta releases should be considered (default: false)

tools/osbuilder/scripts/lib.sh

@@ -300,13 +300,13 @@ detect_libseccomp_info()
 	export LIBSECCOMP_VERSION="$(get_package_version_from_kata_yaml "$libseccomp_ver_yq_path")"
 	export LIBSECCOMP_URL="$(get_package_version_from_kata_yaml "$libseccomp_url_yq_path")"
 
-	info "Get gperf version and url from ${kata_versions_file}"
+	info "Get gperf version from ${kata_versions_file}"
 	local gperf_ver_yq_path="externals.gperf.version"
-	local gperf_url_yq_path="externals.gperf.url"
 	export GPERF_VERSION="$(get_package_version_from_kata_yaml "$gperf_ver_yq_path")"
-	export GPERF_URL="$(get_package_version_from_kata_yaml "$gperf_url_yq_path")"
+	# Note: gperf URLs are now fetched from versions.yaml .externals.gperf.urls array
+	# by ci/install_libseccomp.sh using download_from_mirror_list
 
-	[ -n "$LIBSECCOMP_VERSION" ] && [ -n "$GPERF_VERSION" ] && [ -n "$LIBSECCOMP_URL" ] && [ -n "$GPERF_URL" ]
+	[ -n "$LIBSECCOMP_VERSION" ] && [ -n "$GPERF_VERSION" ] && [ -n "$LIBSECCOMP_URL" ]
 }
 
 before_starting_container() {

tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh

@@ -1112,7 +1112,8 @@ install_agent() {
 	export LIBSECCOMP_VERSION="$(get_from_kata_deps ".externals.libseccomp.version")"
 	export LIBSECCOMP_URL="$(get_from_kata_deps ".externals.libseccomp.url")"
 	export GPERF_VERSION="$(get_from_kata_deps ".externals.gperf.version")"
-	export GPERF_URL="$(get_from_kata_deps ".externals.gperf.url")"
+	# Note: gperf URLs are now fetched from versions.yaml .externals.gperf.urls array
+	# by ci/install_libseccomp.sh using download_from_mirror_list
 
 	info "build static agent"
 	DESTDIR="${destdir}" AGENT_POLICY="${AGENT_POLICY}" "${agent_builder}"
@@ -1215,7 +1216,8 @@ install_tools_helper() {
 	export LIBSECCOMP_VERSION="$(get_from_kata_deps ".externals.libseccomp.version")"
 	export LIBSECCOMP_URL="$(get_from_kata_deps ".externals.libseccomp.url")"
 	export GPERF_VERSION="$(get_from_kata_deps ".externals.gperf.version")"
-	export GPERF_URL="$(get_from_kata_deps ".externals.gperf.url")"
+	# Note: gperf URLs are now fetched from versions.yaml .externals.gperf.urls array
+	# by ci/install_libseccomp.sh using download_from_mirror_list
 
 	info "build static ${tool}"
 	${tools_builder} ${tool}

tools/packaging/scripts/download-with-oras-cache.sh

@@ -365,7 +365,15 @@ download_component() {
 	
 	if [[ -z "${base_url}" ]]; then
 		if command -v yq &>/dev/null; then
-			base_url=$(get_from_kata_deps ".externals.${component}.url")
+			# Try urls array first (for components with multi-mirror support),
+			# then fall back to single url field
+			base_url=$(get_from_kata_deps ".externals.${component}.urls[0]")
+			if [[ -z "${base_url}" ]] || [[ "${base_url}" == "null" ]]; then
+				base_url=$(get_from_kata_deps ".externals.${component}.url")
+			fi
+			# Remove surrounding quotes if present
+			base_url="${base_url%\"}"
+			base_url="${base_url#\"}"
 		else
 			die "Component URL not provided and yq not available. Set ${component_upper}_URL environment variable."
 		fi

tools/packaging/static-build/agent/build.sh

@@ -30,7 +30,6 @@ docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	--env LIBSECCOMP_VERSION=${LIBSECCOMP_VERSION} \
 	--env LIBSECCOMP_URL=${LIBSECCOMP_URL} \
 	--env GPERF_VERSION=${GPERF_VERSION} \
-	--env GPERF_URL=${GPERF_URL} \
 	--env ORAS_CACHE_HELPER="${repo_root_dir}/tools/packaging/scripts/download-with-oras-cache.sh" \
 	--env USE_ORAS_CACHE="${USE_ORAS_CACHE:-yes}" \
 	--env PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-no}" \

tools/packaging/static-build/tools/build.sh

@@ -30,7 +30,6 @@ docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	--env LIBSECCOMP_VERSION=${LIBSECCOMP_VERSION} \
 	--env LIBSECCOMP_URL=${LIBSECCOMP_URL} \
 	--env GPERF_VERSION=${GPERF_VERSION} \
-	--env GPERF_URL=${GPERF_URL} \
 	--env ORAS_CACHE_HELPER="${repo_root_dir}/tools/packaging/scripts/download-with-oras-cache.sh" \
 	--env USE_ORAS_CACHE="${USE_ORAS_CACHE:-yes}" \
 	--env PUSH_TO_REGISTRY="${PUSH_TO_REGISTRY:-no}" \

versions.yaml

@@ -342,7 +342,10 @@ externals:
 
   gperf:
     description: "GNU gperf is a perfect hash function generator"
-    url: "http://ftpmirror.gnu.org/gperf/"
+    urls:
+      - "https://ftp.gnu.org/gnu/gperf/"
+      - "https://mirrors.kernel.org/gnu/gperf/"
+      - "https://ftpmirror.gnu.org/gnu/gperf/"
     version: "3.3"
 
   hadolint: