CI: Test remaining install methods

Add new CI tests to ensure that the following installation methods are
also tested:

- "Automatic" method ([`kata-manager`](https://github.com/kata-containers/tests/tree/master/cmd/kata-manager))
- "Scripted" method ([`kata-doc-to-script`](https://github.com/kata-containers/tests/blob/master/.ci/kata-doc-to-script.sh))

**Note:** the "Automatic" method is **not** the same as the existing
`kata-manager` test: the existing test executes the "Manual"
installation method (which runs `kata-manager` to execute the
appropriate distro-specific install guide). However, this new test
executes the `install/installing-with-kata-manager.md` document, which
subsequently calls the `kata-manager` script.

Since the "Automatic" and "Scripted" installation methods are designed
to run "standalone" (without requiring any local git repo clones), the
script which runs these new tests has to take care to ensure the
environment they run in is clean. It does this by using the following
approach:

- Removes any local Kata github repos from the standard `GOPATH`
  locations (to ensure the scripts do not inadvertently access local
  files) [1].
- Creates a temporary directory containing:
  - A copy of *itself*.
  - The scripts it generated from the "Automatic" and "Scripted" installation documents.
- Re-exec's itself to run the version in the temporary directory,
  passing an option that tells itself to simply execute the scripts in
  the specified directory.
- It then runs the scripts in the directory specified.

---

[1] - Since the recursive delete of all local Kata github repos is
potentially dangerous, the test will immediately fail if the standard
`KATA_DEV_MODE` variable is set (since this denotes a developer system)
and will also fail unless the standard `CI` variable is set (denoting
the script is running in a Continuous Integration environment, such as
JenkinsCI.

Fixes #278.

Signed-off-by: James O. D. Hunt <james.o.hunt@intel.com>

.ci/test-install-docs.sh

@@ -10,6 +10,9 @@ set -e
 # golang locations.
 export GOPATH=${GOPATH:-${HOME}/go}
 
+typeset -r script_name="${0##*/}"
+typeset -r script_dir="$(cd "$(dirname "${0}")" && pwd)"
+
 typeset -r docker_image="busybox"
 typeset -r kata_project_url="github.com/kata-containers"
 typeset -r test_repo="${kata_project_url}/tests"
@@ -18,6 +21,7 @@ typeset -r test_repo_dir="${GOPATH}/src/${test_repo}"
 typeset -r kata_project_dir="${GOPATH}/src/${kata_project_url}"
 
 typeset -r mgr="${test_repo_dir}/cmd/kata-manager/kata-manager.sh"
+typeset -r doc_to_script="${test_repo_dir}/.ci/kata-doc-to-script.sh"
 
 die()
 {
@@ -32,6 +36,52 @@ info()
 	echo "INFO: $msg"
 }
 
+usage()
+{
+	cat <<EOT
+Description: Run Kata documentation CI tests.
+
+Usage: $script_name [options]
+
+Options:
+
+ -h        : Show this help.
+ -t <dir>  : Run all scripts ("\*.sh" files) in the specified
+             directory.
+
+Notes:
+  - The '-t' option is not generally useful - it is used by this
+    script which re-exec's itself with this option.
+
+EOT
+}
+
+# Re-execute the running script from a temporary directory to allow the
+# script to continue executing even if the original source file is deleted.
+reexec_in_tmpdir()
+{
+	local -r test_dir="$1"
+
+	[ -d "${test_dir}" ] || die "invalid test dir: ${test_dir}"
+
+	if [ "${script_dir}" = "${test_dir}" ]
+	then
+		# Already running from temp directory so nothing else to do
+		return
+	fi
+
+	local new
+	new="${test_dir}/${script_name}"
+
+	install --mode 750 "${0}" "${new}"
+
+	info "Re-execing ${0} as ${new}"
+
+	cd "${test_dir}"
+
+	exec "${new}" -t "${test_dir}/tests"
+}
+
 # Grab a copy of the tests repository
 get_tests_repo()
 {
@@ -42,6 +92,26 @@ get_tests_repo()
 	git clone "${test_repo_url}" "${test_repo_dir}"
 }
 
+# Delete all local github repo clones.
+#
+# This is required to ensure that the tests themselves (re-)create these
+# clones.
+delete_kata_repos()
+{
+	[ -n "${KATA_DEV_MODE}" ] && die "Not continuing as this is a dev system"
+	[ -z "${CI}" ] && die "Not continuing as this is a non-CI environment"
+
+	local cwd="$PWD"
+
+	info "Deleting all local kata repositories below ${kata_project_dir}"
+
+	[ -d "${kata_project_dir}" ] && rm -rf "${kata_project_dir}" || true
+
+	# Recreate the empty directory, taking care to handle the scenario
+	# where the script is run from within the just-deleted directory.
+	mkdir -p "$cwd" && cd "$cwd"
+}
+
 setup()
 {
 	source /etc/os-release || source /usr/lib/os-release
@@ -82,11 +152,81 @@ test_distro_install_guide()
 	info "Install using ${test_name}"
 
 	create_kata_container "${test_name}"
+
+	# Clean up
+	$mgr remove-packages
+}
+
+# Apart from the distro-specific install guides, users can choose to install
+# using one of the following methods:
+#
+# - kata-manager ("Automatic" method).
+# - kata-doc-to-script ("Scripted" method).
+#
+# Testing these is awkward because we need to "execute" the documents
+# describing those install methods, but since those install methods should
+# themselves entirely document/handle an installation method, we need to
+# convert each install document to a script, then delete all the kata code
+# repositories. This ensures that when each install method script is run, it
+# does not rely on any local files (it should download anything it needs). But
+# since we're deleting the repos, we need to copy this script to a temporary
+# location, along with the install scripts this function generates, and then
+# re-exec this script with an option to ask it to run the scripts the previous
+# instance of this script just generated.
+test_alternative_install_methods()
+{
+	local -a files
+	files+=("installing-with-kata-manager.md")
+	files+=("installing-with-kata-doc-to-script.md")
+
+	local tmp_dir
+
+	tmp_dir=$(mktemp -d)
+
+	local script_file
+
+	local file
+
+	local tests_dir
+	tests_dir="${tmp_dir}/tests"
+
+	mkdir -p "${tests_dir}"
+
+	local -i num=0
+
+	# Convert the docs to scripts
+	for file in "${files[@]}"
+	do
+		num+=1
+
+		local file_path
+		local script_file
+		local script_file_path
+		local test_name
+
+		file_path="${script_dir}/../install/${file}"
+		script_file=${file/.md/.sh}
+
+		# Add a numeric prefix so the tests are run in the array order 
+		test_name=$(printf "%.2d-%s" "${num}" "${script_file}")
+
+		script_file_path="${tests_dir}/${test_name}"
+
+		info "Creating test script ${test_name} from ${file}"
+
+		bash "${doc_to_script}" "${file_path}" "${script_file_path}"
+	done
+
+	reexec_in_tmpdir "${tmp_dir}"
+
+	# Not reached
+	die "re-exec failed"
 }
 
 run_tests()
 {
 	test_distro_install_guide
+	test_alternative_install_methods
 }
 
 # Detect if any installation documents changed. If so, execute all the
@@ -141,9 +281,67 @@ check_install_docs()
 	run_tests
 }
 
+# Run the test scripts in the specified directory.
+run_tests_from_dir()
+{
+	local -r test_dir="$1"
+
+	[ -e "$test_dir" ] || die "invalid test dir: ${test_dir}"
+
+	cd "${test_dir}"
+
+	info "Looking for tests scripts to run in directory ${test_dir}"
+
+	for t in $(ls -- *.sh)
+	do
+		# Ensure the test script cannot access any local files
+		# (since it should be standalone and download any files
+		# it needs).
+		delete_kata_repos
+
+		info "Running test script '$t'"
+		bash -x "${t}"
+
+		# Ensure it is possible to use the installed system
+		create_kata_container "${t}"
+
+		# Re-run setup to recreate the tests repo that was deleted
+		# before the test ran.
+		setup
+
+		# Packaged install so clean up
+		# (Note that '$mgr' should now exist again)
+		$mgr remove-packages
+	done
+
+	# paranoia
+	[ -d "${test_dir}" ] && rm -rf "${test_dir}"
+
+	info "All tests passed"
+}
+
 main()
 {
+	local opt
+	local test_dir
+
 	setup
+
+	while getopts "ht:" opt
+	do
+		case "$opt" in
+			h) usage; exit 0;;
+			t) test_dir="$OPTARG";;
+			*) die "invalid option: $opt";;
+		esac
+	done
+
+	if [ -n "$test_dir" ]
+	then
+		run_tests_from_dir "$test_dir"
+		exit 0
+	fi
+
 	check_install_docs
 }