Merge pull request #83 from jodh-intel/create-summary-file

refactor and create a summary file inside the image

.ci/setup.sh

@@ -14,12 +14,13 @@ bash "${cidir}/static-checks.sh"
 source /etc/os-release
 
 if [ "$ID" == fedora ];then
-	sudo -E dnf -y install automake  bats
+	sudo -E dnf -y install automake bats yamllint
 elif [ "$ID" == ubuntu ];then
 	#bats isn't available for Ubuntu trusty, need for travis
 	sudo add-apt-repository -y ppa:duggan/bats
 	sudo apt-get -qq update
-	sudo apt-get install -y -qq automake bats qemu-utils
+	sudo apt-get install -y -qq automake bats qemu-utils python-pip
+	sudo pip install yamllint
 else 
 	echo "Linux distribution not supported"
 fi

Makefile

@@ -10,10 +10,16 @@ DISTRO_ROOTFS := "$(PWD)/$(DISTRO)_rootfs"
 IMG_SIZE=500
 AGENT_INIT ?= no
 
+VERSION_FILE := ./VERSION
+VERSION := $(shell grep -v ^\# $(VERSION_FILE))
+COMMIT_NO := $(shell git rev-parse HEAD 2> /dev/null || true)
+COMMIT := $(if $(shell git status --porcelain --untracked-files=no),${COMMIT_NO}-dirty,${COMMIT_NO})
+VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION))
+
 all: rootfs image initrd
 rootfs:
 	@echo Creating rootfs based on "$(DISTRO)"
-	"$(MK_DIR)/rootfs-builder/rootfs.sh" -r "$(DISTRO_ROOTFS)" "$(DISTRO)"
+	"$(MK_DIR)/rootfs-builder/rootfs.sh" -o $(VERSION_COMMIT) -r "$(DISTRO_ROOTFS)" "$(DISTRO)"
 
 image: rootfs image-only
 

VERSION

@@ -0,0 +1,2 @@
+# This is the version of osbuilder.
+0.0.1

image-builder/image_builder.sh

@@ -6,48 +6,23 @@
 
 set -e
 
+[ -n "$DEBUG" ] && set -x
+
 script_name="${0##*/}"
 script_dir="$(dirname $(readlink -f $0))"
 
-if [ -n "$DEBUG" ] ; then
+lib_file="${script_dir}/../scripts/lib.sh"
-	set -x
+source "$lib_file"
-fi
 
-SCRIPT_NAME="${0##*/}"
 IMAGE="${IMAGE:-kata-containers.img}"
 AGENT_BIN=${AGENT_BIN:-kata-agent}
 AGENT_INIT=${AGENT_INIT:-no}
 
-die()
-{
-	local msg="$*"
-	echo "ERROR: ${msg}" >&2
-	exit 1
-}
-
-OK()
-{
-	local msg="$*"
-	echo "[OK] ${msg}" >&2
-}
-
-info()
-{
-	local msg="$*"
-	echo "INFO: ${msg}"
-}
-
-warning()
-{
-        local msg="$*"
-        echo "WARNING: ${msg}"
-}
-
 usage()
 {
 	error="${1:-0}"
 	cat <<EOT
-Usage: ${SCRIPT_NAME} [options] <rootfs-dir>
+Usage: ${script_name} [options] <rootfs-dir>
 	This script will create a Kata Containers image file of
         an adequate size based on the <rootfs-dir> directory.
         The size of the image can be also be specified manually
@@ -129,6 +104,7 @@ if [ -n "${USE_DOCKER}" ] ; then
 		--env AGENT_INIT=${AGENT_INIT} \
 		-v /dev:/dev \
 		-v "${script_dir}":"/osbuilder" \
+		-v "${script_dir}/../scripts":"/scripts" \
 		-v "${ROOTFS}":"/rootfs" \
 		-v "${IMAGE_DIR}":"/image" \
 		${image_name} \
@@ -138,7 +114,7 @@ if [ -n "${USE_DOCKER}" ] ; then
 fi
 # The kata rootfs image expect init and kata-agent to be installed
 init="${ROOTFS}/sbin/init"
-[ -x "${init}" ] || [ -L ${init} ] || die "/sbin/init is not installed in ${ROOTFS_DIR}"
+[ -x "${init}" ] || [ -L ${init} ] || die "/sbin/init is not installed in ${ROOTFS}"
 OK "init is installed"
 [ "${AGENT_INIT}" == "yes" ] || [ -x "${ROOTFS}/usr/bin/${AGENT_BIN}" ] || \
 	die "/usr/bin/${AGENT_BIN} is not installed in ${ROOTFS}

initrd-builder/initrd_builder.sh

@@ -6,42 +6,23 @@
 
 set -e
 
+[ -n "$DEBUG" ] && set -x
+
 script_name="${0##*/}"
 script_dir="$(dirname $(readlink -f $0))"
 
-if [ -n "$DEBUG" ] ; then
+lib_file="${script_dir}/../scripts/lib.sh"
-	set -x
+source "$lib_file"
-fi
 
-SCRIPT_NAME="${0##*/}"
 INITRD_IMAGE="${INITRD_IMAGE:-kata-containers-initrd.img}"
 AGENT_BIN=${AGENT_BIN:-kata-agent}
 AGENT_INIT=${AGENT_INIT:-no}
 
-die()
-{
-	local msg="$*"
-	echo "ERROR: ${msg}" >&2
-	exit 1
-}
-
-OK()
-{
-	local msg="$*"
-	echo "[OK] ${msg}" >&2
-}
-
-info()
-{
-	local msg="$*"
-	echo "INFO: ${msg}"
-}
-
 usage()
 {
 	error="${1:-0}"
 	cat <<EOT
-Usage: ${SCRIPT_NAME} [options] <rootfs-dir>
+Usage: ${script_name} [options] <rootfs-dir>
 	This script creates a Kata Containers initrd image file based on the
 	<rootfs-dir> directory.
 
@@ -55,9 +36,6 @@ Extra environment variables:
 		    DEFAULT: kata-agent
 	AGENT_INIT: use kata agent as init process
 		    DEFAULT: no
-	USE_DOCKER: If set, the image builds in a Docker Container. Setting
-		    this variable requires Docker.
-	            DEFAULT: not set
 EOT
 exit "${error}"
 }

rootfs-builder/clearlinux/config.sh

@@ -4,12 +4,16 @@
 # SPDX-License-Identifier: Apache-2.0
 
 OS_NAME="Clear"
+REPO_NAME="clear"
 
 OS_VERSION=${OS_VERSION:-latest}
 
-BASE_URL="https://download.clearlinux.org/current/${ARCH}/os/"
+clr_url="https://download.clearlinux.org"
 
-REPO_NAME="clear"
+# resolve version
+[ "${OS_VERSION}" = "latest" ] && OS_VERSION=$(curl -sL "${clr_url}/latest")
+
+BASE_URL="${clr_url}/releases/${OS_VERSION}/${REPO_NAME}/${ARCH}/os/"
 
 PACKAGES="iptables-bin libudev0-shim"
 

rootfs-builder/rootfs.sh

@@ -6,6 +6,8 @@
 
 set -e
 
+[ -n "$DEBUG" ] && set -x
+
 script_name="${0##*/}"
 script_dir="$(dirname $(readlink -f $0))"
 AGENT_VERSION=${AGENT_VERSION:-master}
@@ -14,24 +16,25 @@ AGENT_BIN=${AGENT_BIN:-kata-agent}
 AGENT_INIT=${AGENT_INIT:-no}
 KERNEL_MODULES_DIR=${KERNEL_MODULES_DIR:-""}
 
+lib_file="${script_dir}/../scripts/lib.sh"
+source "$lib_file"
+
 # Default architecture
 ARCH=${ARCH:-"x86_64"}
 
-#Load default vesions for golang and other componets
+# Load default versions for golang and other componets
 source "${script_dir}/versions.txt"
 
-# config file
+# distro-specific config file
 typeset -r CONFIG_SH="config.sh"
 
-# Name of the extra file that could implement build_rootfs
+# Name of an optional distro-specific file which, if it exists, must implement the
+# build_rootfs() function.
 typeset -r LIB_SH="rootfs_lib.sh"
 
-if [ -n "$DEBUG" ] ; then
+#$1: Error code if want to exit different to 0
-	set -x
+usage()
-fi
+{
-
-#$1: Error code if want to exit differnt to 0
-usage(){
 	error="${1:-0}"
 	cat <<EOT
 USAGE: Build a Guest OS rootfs for Kata Containers image
@@ -46,6 +49,7 @@ $(get_distros)
 Options:
 -a  : agent version DEFAULT: ${AGENT_VERSION} ENV: AGENT_VERSION 
 -h  : Show this help message
+-o  : specify version of osbuilder
 -r  : rootfs directory DEFAULT: ${ROOTFS_DIR} ENV: ROOTFS_DIR
 
 ENV VARIABLES:
@@ -62,25 +66,6 @@ EOT
 exit "${error}"
 }
 
-die()
-{
-	msg="$*"
-	echo "ERROR: ${msg}" >&2
-	exit 1
-}
-
-info()
-{
-	msg="$*"
-	echo "INFO: ${msg}" >&2
-}
-
-OK()
-{
-	msg="$*"
-	echo "INFO: [OK] ${msg}" >&2
-}
-
 get_distros() {
 	cdirs=$(find "${script_dir}" -maxdepth 1 -type d)
 	find ${cdirs} -maxdepth 1 -name "${CONFIG_SH}" -printf '%H\n' | while read dir; do
@@ -88,13 +73,14 @@ get_distros() {
 	done
 }
 
-
+check_function_exist()
-check_function_exist() {
+{
 	function_name="$1"
 	[ "$(type -t ${function_name})" == "function" ] || die "${function_name} function was not defined"
 }
 
-generate_dockerfile() {
+generate_dockerfile()
+{
 	dir="$1"
 
 	case "$(arch)" in
@@ -130,31 +116,43 @@ ENV PATH=\$PATH:\$GOROOT/bin:\$GOPATH/bin
 	popd
 }
 
-setup_agent_init() {
+setup_agent_init()
+{
 	agent_bin="$1"
 	init_bin="$2"
+
+	[ -z "$agent_bin" ] && die "need agent binary path"
+	[ -z "$init_bin" ] && die "need init bin path"
+
 	info "Install $agent_bin as init process"
 	mv -f "${agent_bin}" ${init_bin}
 	OK "Agent is installed as init process"
 }
 
-copy_kernel_modules() {
+copy_kernel_modules()
-	local module_dir=$1
+{
-	local rootfs_dir=$2
+	local module_dir="$1"
+	local rootfs_dir="$2"
 
-	[ -z "module_dir" -o -z "rootfs_dir" ] && die "module dir and rootfs dir must be specified"
+	[ -z "$module_dir" ] && die "need module directory"
+	[ -z "$rootfs_dir" ] && die "need rootfs directory"
+
+	local destdir="${rootfs_dir}/lib/modules"
 
 	info "Copy kernel modules from ${KERNEL_MODULES_DIR}"
-	mkdir -p ${rootfs_dir}/lib/modules/
+	mkdir -p "${destdir}"
-	cp -a ${KERNEL_MODULES_DIR} ${rootfs_dir}/lib/modules/
+	cp -a "${KERNEL_MODULES_DIR}" "${dest_dir}/"
 	OK "Kernel modules copied"
 }
 
-while getopts c:hr: opt
+OSBUILDER_VERSION="unknown"
+
+while getopts c:ho:r: opt
 do
 	case $opt in
 		a)	AGENT_VERSION="${OPTARG}" ;;
 		h)	usage ;;
+		o)	OSBUILDER_VERSION="${OPTARG}" ;;
 		r)	ROOTFS_DIR="${OPTARG}" ;;
 	esac
 done
@@ -167,6 +165,8 @@ shift $(($OPTIND - 1))
 
 [ -n "${KERNEL_MODULES_DIR}" ] && [ ! -d "${KERNEL_MODULES_DIR}" ] && die "KERNEL_MODULES_DIR defined but is not an existing directory"
 
+[ -z "${OSBUILDER_VERSION}" ] && die "need osbuilder version"
+
 distro="$1"
 
 [ -n "${distro}" ] || usage 1
@@ -176,10 +176,6 @@ distro_config_dir="${script_dir}/${distro}"
 rootfs_config="${distro_config_dir}/${CONFIG_SH}"
 source "${rootfs_config}"
 
-lib_file="${script_dir}/../scripts/lib.sh"
-info "Source $lib_file"
-[ -e "$lib_file" ] && source "$lib_file" || true
-
 [ -d "${distro_config_dir}" ] || die "Not found configuration directory ${distro_config_dir}"
 
 if [ -z "$ROOTFS_DIR" ]; then
@@ -224,6 +220,7 @@ if [ -n "${USE_DOCKER}" ] ; then
 		--env GOPATH="${GOPATH}" \
 		--env KERNEL_MODULES_DIR="${KERNEL_MODULES_DIR}" \
 		--env EXTRA_PKGS="${EXTRA_PKGS}" \
+		--env OSBUILDER_VERSION="${OSBUILDER_VERSION}" \
 		-v "${script_dir}":"/osbuilder" \
 		-v "${ROOTFS_DIR}":"/rootfs" \
 		-v "${script_dir}/../scripts":"/scripts" \
@@ -250,11 +247,17 @@ make clean
 make INIT=${AGENT_INIT}
 make install DESTDIR="${ROOTFS_DIR}" INIT=${AGENT_INIT}
 popd
-[ -x "${ROOTFS_DIR}/usr/bin/${AGENT_BIN}" ] || die "/usr/bin/${AGENT_BIN} is not installed in ${ROOTFS_DIR}"
+
+AGENT_DIR="${ROOTFS_DIR}/usr/bin"
+AGENT_DEST="${AGENT_DIR}/${AGENT_BIN}"
+[ -x "${AGENT_DEST}" ] || die "${AGENT_DEST} is not installed in ${ROOTFS_DIR}"
 OK "Agent installed"
 
-[ "${AGENT_INIT}" == "yes" ] && setup_agent_init "${ROOTFS_DIR}/usr/bin/${AGENT_BIN}" "${init}"
+[ "${AGENT_INIT}" == "yes" ] && setup_agent_init "${AGENT_DEST}" "${init}"
 
 info "Check init is installed"
 [ -x "${init}" ] || [ -L "${init}" ] || die "/sbin/init is not installed in ${ROOTFS_DIR}"
 OK "init is installed"
+
+info "Creating summary file"
+create_summary_file "${ROOTFS_DIR}"

scripts/lib.sh

@@ -6,7 +6,33 @@
 
 set -e
 
-check_program(){
+die()
+{
+	local msg="$*"
+	echo "ERROR: ${msg}" >&2
+	exit 1
+}
+
+OK()
+{
+	local msg="$*"
+	echo "[OK] ${msg}" >&2
+}
+
+info()
+{
+	local msg="$*"
+	echo "INFO: ${msg}"
+}
+
+warning()
+{
+	local msg="$*"
+	echo "WARNING: ${msg}"
+}
+
+check_program()
+{
 	type "$1" >/dev/null 2>&1
 }
 
@@ -36,7 +62,7 @@ reposdir=/root/mash
 retries=5
 EOF
 	if [ "$BASE_URL" != "" ]; then
-            cat >> "${DNF_CONF}" << EOF
+		cat >> "${DNF_CONF}" << EOF
 
 [base]
 name=${OS_NAME}-${OS_VERSION} ${REPO_NAME}
@@ -45,7 +71,7 @@ baseurl=${BASE_URL}
 enabled=1
 EOF
 	elif [ "$MIRROR_LIST" != "" ]; then
-	    cat >> "${DNF_CONF}" << EOF
+		cat >> "${DNF_CONF}" << EOF
 
 [base]
 name=${OS_NAME}-${OS_VERSION} ${REPO_NAME}
@@ -55,13 +81,12 @@ EOF
 	fi
 
 	if [ "$GPG_KEY_FILE" != "" ]; then
-            cat >> "${DNF_CONF}" << EOF
+		cat >> "${DNF_CONF}" << EOF
 gpgcheck=1
 gpgkey=file://${CONFIG_DIR}/${GPG_KEY_FILE}
 
 EOF
 	fi
-
 }
 
 build_rootfs()
@@ -69,6 +94,8 @@ build_rootfs()
 	# Mandatory
 	local ROOTFS_DIR="$1"
 
+	[ -z "$ROOTFS_DIR" ] && die "need rootfs"
+
 	# In case of support EXTRA packages, use it to allow
 	# users add more packages to the base rootfs
 	local EXTRA_PKGS=${EXTRA_PKGS:-""}
@@ -99,3 +126,65 @@ build_rootfs()
 
 	[ -n "${ROOTFS_DIR}" ]  && rm -r "${ROOTFS_DIR}${CACHE_DIR}"
 }
+
+# Create a YAML metadata file inside the rootfs.
+#
+# This provides useful information about the rootfs than can be interrogated
+# once the rootfs has been converted into a image/initrd.
+create_summary_file()
+{
+	local -r rootfs_dir="$1"
+
+	[ -z "$rootfs_dir" ] && die "need rootfs"
+
+	local -r file_dir="/var/lib/osbuilder"
+	local -r dir="${rootfs_dir}${file_dir}"
+
+	local -r filename="osbuilder.yaml"
+	local file="${dir}/${filename}"
+
+	local -r now=$(date '+%Y-%m-%dT%T.%N%zZ')
+
+	# sanitise package list
+	PACKAGES=$(echo "$PACKAGES"|tr ' ' '\n'|sort -u|tr '\n' ' ')
+
+	local -r packages=$(for pkg in ${PACKAGES}; do echo "    - \"${pkg}\""; done)
+
+	mkdir -p "$dir"
+
+	# Semantic version of the summary file format.
+	#
+	# XXX: Increment every time the format of the summary file changes!
+	local -r format_version="0.0.1"
+
+	local -r osbuilder_url="https://github.com/kata-containers/osbuilder"
+
+	local agent="${AGENT_DEST}"
+	[ "$AGENT_INIT" = yes ] && agent="${init}"
+
+	local -r agent_version=$("$agent" --version|awk '{print $NF}')
+
+	cat >"$file"<<-EOT
+	---
+	osbuilder:
+	  url: "${osbuilder_url}"
+	  version: "${OSBUILDER_VERSION}"
+	rootfs-creation-time: "${now}"
+	description: "osbuilder rootfs"
+	file-format-version: "${format_version}"
+	architecture: "${ARCH}"
+	base-distro:
+	  name: "${OS_NAME}"
+	  version: "${OS_VERSION}"
+	  packages:
+${packages}
+	agent:
+	  url: "https://${GO_AGENT_PKG}"
+	  name: "${AGENT_BIN}"
+	  version: "${agent_version}"
+	  agent-is-init-daemon: "${AGENT_INIT}"
+EOT
+
+	local rootfs_file="${file_dir}/$(basename "${file}")"
+	info "Created summary file '${rootfs_file}' inside rootfs"
+}

tests/image_creation.bats

@@ -27,7 +27,12 @@ teardown(){
 
 function build_rootfs()
 {
+	local file="/var/lib/osbuilder/osbuilder.yaml"
+	local full="${tmp_rootfs}${file}"
+
 	sudo -E ${rootfs_sh} -r "${tmp_rootfs}" "${distro}"
+
+	yamllint "${full}"
 }
 
 function build_image()