Merge pull request #191 from marcov/allow-failures

tests: allow rootfs build to fail for specific distros

rootfs-builder/euleros/config.sh

@@ -22,3 +22,6 @@ INIT_PROCESS=systemd
 # List of zero or more architectures to exclude from build,
 # as reported by  `uname -m`
 ARCH_EXCLUDE_LIST=()
+# Allow the build to fail without generating an error.
+# For more info see: https://github.com/kata-containers/osbuilder/issues/190
+BUILD_CAN_FAIL=1

rootfs-builder/rootfs.sh

@@ -15,6 +15,7 @@ GO_AGENT_PKG=${GO_AGENT_PKG:-github.com/kata-containers/agent}
 AGENT_BIN=${AGENT_BIN:-kata-agent}
 AGENT_INIT=${AGENT_INIT:-no}
 KERNEL_MODULES_DIR=${KERNEL_MODULES_DIR:-""}
+OSBUILDER_VERSION="unknown"
 
 lib_file="${script_dir}/../scripts/lib.sh"
 source "$lib_file"
@@ -40,35 +41,64 @@ usage()
 {
 	error="${1:-0}"
 	cat <<EOT
-USAGE: Build a Guest OS rootfs for Kata Containers image
-${script_name} [options] <distro_name>
 
-<distro_name> : Linux distribution to use as base OS.
+Usage: ${script_name} [options] <distro>
 
-Supported Linux distributions:
+Build a rootfs based on <distro> OS, to be included in a Kata Containers
+image.
 
-$(get_distros)
+Supported <distro> values:
-
+$(get_distros | tr "\n" " ")
-Refer the Platform-OS Compatibility Matrix: https://github.com/kata-containers/osbuilder#platform-distro-compatibility-matrix
 
 Options:
--a  : agent version DEFAULT: ${AGENT_VERSION} ENV: AGENT_VERSION
+  -a <version>      Specify the agent version. Overrides the AGENT_VERSION
--h  : show this help message
+                    environment variable.
--l  : list the supported Linux distributions
+  -h                Show this help message.
--o  : specify version of osbuilder
+  -l                List the supported Linux distributions and exit immediately.
--r  : rootfs directory DEFAULT: ${ROOTFS_DIR} ENV: ROOTFS_DIR
+  -o <version>      Specify the version of osbuilder to embed in the rootfs
--t  : print the test config for a given <distro_name>
+                    yaml description.
+  -r <directory>    Specify the rootfs base directory. Overrides the ROOTFS_DIR
+                    environment variable.
+  -t                Print the test configuration for <distro> and exit
+                    immediately.
+
+Environment Variables:
+AGENT_BIN           Name of the agent binary (used when running sanity checks on
+                    the rootfs).
+                    Default value: ${AGENT_BIN}
+
+AGENT_INIT          When set to "yes", use ${AGENT_BIN} as init process in place
+                    of systemd.
+                    Default value: no
+
+AGENT_VERSION       Version of the agent to include in the rootfs.
+                    Default value: ${AGENT_VERSION:-<not set>}
+
+GO_AGENT_PKG        URL of the Git repository hosting the agent package.
+                    Default value: ${GO_AGENT_PKG}
+
+GRACEFUL_EXIT       If set, and if the <distro> configuration specifies a
+                    non-empty BUILD_CAN_FAIL variable, do not return with an
+                    error code in case any of the build step fails.
+                    This is used when running CI jobs, to tolerate failures for
+                    specific distributions.
+                    Default value: <not set>
+
+KERNEL_MODULES_DIR  Path to a directory containing kernel modules to include in
+                    the rootfs.
+                    Default value: <empty>
+
+ROOTFS_DIR          Path to the directory that is populated with the rootfs.
+                    Default value: <${script_name} path>/rootfs-<distro-name>
+
+USE_DOCKER          If set, build the rootfs inside a container (requires
+                    Docker).
+                    Default value: <not set>
+
+Refer to the Platform-OS Compatibility Matrix for more details on the supported
+architectures:
+https://github.com/kata-containers/osbuilder#platform-distro-compatibility-matrix
 
-ENV VARIABLES:
-GO_AGENT_PKG: Change the golang package url to get the agent source code
-            DEFAULT: ${GO_AGENT_PKG}
-AGENT_BIN   : Name of the agent binary (needed to check if agent is installed)
-USE_DOCKER: If set will build rootfs in a Docker Container (requries docker)
-            DEFAULT: not set
-AGENT_INIT  : Use ${AGENT_BIN} as init process.
-            DEFAULT: no
-KERNEL_MODULES_DIR: Optional kernel modules to put into the rootfs.
-		    DEFAULT: ""
 EOT
 exit "${error}"
 }
@@ -189,7 +219,16 @@ copy_kernel_modules()
 	OK "Kernel modules copied"
 }
 
-OSBUILDER_VERSION="unknown"
+error_handler()
+{
+	[ "$?" -eq 0 ] && return
+
+	if [ -n "$GRACEFUL_EXIT" ] && [ -n "$BUILD_CAN_FAIL" ]; then
+		info "Detected a build error, but $distro is allowed to fail (BUILD_CAN_FAIL specified), so exiting sucessfully"
+		touch "$(dirname ${ROOTFS_DIR})/${distro}_fail"
+		exit 0
+	fi
+}
 
 while getopts a:hlo:r:t: opt
 do
@@ -248,6 +287,11 @@ fi
 CONFIG_DIR=${distro_config_dir}
 check_function_exist "build_rootfs"
 
+if [ -z "$INSIDE_CONTAINER" ] ; then
+	# Capture errors, but only outside of the docker container
+	trap error_handler ERR
+fi
+
 if [ -n "${USE_DOCKER}" ] ; then
 	image_name="${distro}-rootfs-osbuilder"
 
@@ -281,6 +325,7 @@ if [ -n "${USE_DOCKER}" ] ; then
 		--env KERNEL_MODULES_DIR="${KERNEL_MODULES_DIR}" \
 		--env EXTRA_PKGS="${EXTRA_PKGS}" \
 		--env OSBUILDER_VERSION="${OSBUILDER_VERSION}" \
+		--env INSIDE_CONTAINER=1 \
 		-v "${script_dir}":"/osbuilder" \
 		-v "${ROOTFS_DIR}":"/rootfs" \
 		-v "${script_dir}/../scripts":"/scripts" \

rootfs-builder/template/config_template.sh

@@ -17,3 +17,6 @@ INIT_PROCESS=systemd
 # List of zero or more architectures to exclude from build,
 # as reported by  `uname -m`
 ARCH_EXCLUDE_LIST=()
+# [When uncommented,] Allow the build to fail without generating an error
+# For more info see: https://github.com/kata-containers/osbuilder/issues/190
+#BUILD_CAN_FAIL=1

tests/test_config.sh

@@ -3,15 +3,12 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-
+# List of distros not to test, when running all tests with test_images.sh
-if [ -n "${CI:-}" ]; then
+typeset -a skipWhenTestingAll
-	# "Not testing eurleros on Jenkins or Travis:
-	# (unreliable mirros, see: https://github.com/kata-containers/osbuilder/issues/182)
-	# (timeout, see: https://github.com/kata-containers/osbuilder/issues/46)"
-	skipWhenTestingAll=(euleros)
-fi
 
 if [ -n "${TRAVIS:-}" ]; then
-	skipWhenTestingAll+=()
+	# (travis may timeout with euleros, see:
+	#  https://github.com/kata-containers/osbuilder/issues/46)"
+	skipWhenTestingAll+=(euleros)
 fi
 

tests/test_images.sh

@@ -22,6 +22,8 @@ readonly test_config=${script_dir}/test_config.sh
 readonly rootfs_builder=${script_dir}/../rootfs-builder/rootfs.sh
 readonly RUNTIME=${RUNTIME:-kata-runtime}
 readonly MACHINE_TYPE=`uname -m`
+readonly CI=${CI:-}
+readonly ci_results_dir="/var/osbuilder/tests"
 
 # all distro tests must have this prefix
 readonly test_func_prefix="test_distro_"
@@ -221,6 +223,11 @@ setup()
 	[ -z "$images_dir" ] && die "need images directory"
 	mkdir -p "${images_dir}"
 
+	if [ -n "$CI" ]; then
+		sudo -E rm -rf ${ci_results_dir}
+		sudo -E mkdir -p ${ci_results_dir}
+	fi
+
 	export USE_DOCKER=true
 
 	# Travis doesn't support VT-x
@@ -258,7 +265,7 @@ get_distros_config()
 	for d in ${distrosList[@]}; do
 		debug "Getting config for distro $d"
 		distroPattern="\<${d}\>"
-		if [[ "${skipWhenTestingAll[@]}" =~ $distroPattern ]]; then
+		if [[ "${skipWhenTestingAll[@]:-}" =~ $distroPattern ]]; then
 			info "Skipping distro $d as specified by $(basename ${test_config})"
 			continue
 		fi
@@ -360,7 +367,7 @@ call_make() {
 	done
 
 	makeJobs=
-	if [ -z "${CI:-}" ]; then
+	if [ -z "$CI" ]; then
 	  ((makeJobs=$(nproc) / 2))
 	fi
 
@@ -385,7 +392,7 @@ make_initrd() {
 }
 
 get_rootfs_size() {
-	[ $# -ne 1 ] && die "get_rootfs_size with wrong invalid argument"
+	[ $# -ne 1 ] && die "get_rootfs_size: wrong number of arguments"
 
 	local rootfs_dir=$1
 	! [ -d "$rootfs_dir" ] && die "$rootfs_dir is not a valid rootfs path"
@@ -393,6 +400,18 @@ get_rootfs_size() {
 	sudo -E du -sb "${rootfs_dir}" | awk '{print $1}'
 }
 
+
+show_rootfs_metadata() {
+	[ $# -ne 1 ] && die "show_rootfs_metadata: wrong number of arguments"
+	local rootfs_path=$1
+	local osbuilder_file_fullpath="${rootfs_path}/${osbuilder_file}"
+	echo -e "$separator"
+	yamllint "${osbuilder_file_fullpath}"
+
+	info "osbuilder metadata file for $d:"
+	cat "${osbuilder_file_fullpath}" >&2
+}
+
 # Create an image and/or initrd for the available distributions,
 # then test each by configuring the runtime and creating a container.
 #
@@ -408,6 +427,10 @@ test_distros()
 	local distro="$1"
 	get_distros_config "$distro"
 	local separator="~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
+	local commonMakeVars=( \
+		USE_DOCKER=true \
+		ROOTFS_BUILD_DEST="$tmp_rootfs" \
+		IMAGES_BUILD_DEST="$images_dir" )
 
 	echo -e "$separator"
 
@@ -429,6 +452,8 @@ test_distros()
 
 	else
 		info "Running tests for all distros"
+		# Graceful exit allowed for selected distros, but only when testing all distros
+		commonMakeVars+=(GRACEFUL_EXIT=1)
 	fi
 
 	# distro with systemd as init    -> normal rootfs image
@@ -437,11 +462,6 @@ test_distros()
 	# If user does not need rootfs images, then do not build systemd rootfses
 	[ -z "$build_images" ] && distrosSystemd=()
 
-	commonMakeVars=( \
-		USE_DOCKER=true \
-		ROOTFS_BUILD_DEST="$tmp_rootfs" \
-		IMAGES_BUILD_DEST="$images_dir" )
-
 	# Build systemd and agent rootfs with 2 separate jobs
 	bgJobs=()
 
@@ -462,18 +482,6 @@ test_distros()
 		wait $j || die "Background build job failed"
 	done
 
-
-	for d in ${distrosSystemd[@]} ${distrosAgent[@]}; do
-		local rootfs_path="${tmp_rootfs}/${d}_rootfs"
-		osbuilder_file_fullpath="${rootfs_path}/${osbuilder_file}"
-		echo -e "$separator"
-		yamllint "${osbuilder_file_fullpath}"
-
-		info "osbuilder metadata file for $d:"
-		cat "${osbuilder_file_fullpath}" >&2
-	done
-
-
 	# TODO: once support for rootfs images with kata-agent as init is in place,
 	# uncomment the following line
 #	for d in ${distrosSystemd[@]} ${distrosAgent[@]}; do
@@ -482,6 +490,14 @@ test_distros()
 		local image_path="${images_dir}/kata-containers-image-$d.img"
 		local rootfs_size=$(get_rootfs_size "$rootfs_path")
 
+		# Skip failed distros
+		if [ -e "${tmp_rootfs}/${d}_fail" ]; then
+			info "Building rootfs for ${d} failed, not creating an image"
+			[ -n "$CI" ] && sudo -E touch "${ci_results_dir}/${d}_fail"
+			continue
+		fi
+
+		show_rootfs_metadata "$rootfs_path"
 		echo -e "$separator"
 		info "Making rootfs image for ${d}"
 		make_image ${commonMakeVars[@]} $d
@@ -498,6 +514,13 @@ test_distros()
 		local initrd_path="${images_dir}/kata-containers-initrd-$d.img"
 		local rootfs_size=$(get_rootfs_size "$rootfs_path")
 
+		# Skip failed distros
+		if [ -e "${tmp_rootfs}/${d}_fail" ]; then
+			info "Building rootfs for ${d} failed, not creating an initrd"
+			[ -n "$CI" ] && touch "${ci_results_dir}/${d}_fail"
+			continue
+		fi
+
 		echo -e "$separator"
 		info "Making initrd image for ${d}"
 		make_initrd ${commonMakeVars[@]} AGENT_INIT=yes $d