Merge pull request #49 from bergwolf/agent_init

Allow to use agent as init process and create initrd image based on rootfs

.travis.yml

@@ -9,6 +9,10 @@ dist: trusty
 
 language: bash
 
+env:
+  - AGENT_INIT=no
+  - AGENT_INIT=yes
+
 services:
   - docker
 

Makefile

@@ -8,9 +8,16 @@ MK_DIR :=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 DISTRO ?= centos
 DISTRO_ROOTFS := "$(PWD)/$(DISTRO)_rootfs"
 IMG_SIZE=500
+AGENT_INIT ?= no
 
-image:
+rootfs:
 	@echo Creating rootfs based on "$(DISTRO)"
 	"$(MK_DIR)/rootfs-builder/rootfs.sh" -r "$(DISTRO_ROOTFS)" "$(DISTRO)"
+
+image: rootfs
 	@echo Creating image based on "$(DISTRO_ROOTFS)"
-	AGENT_BIN="$(AGENT_BIN)" "$(MK_DIR)/image-builder/image_builder.sh" -s "$(IMG_SIZE)" "$(DISTRO_ROOTFS)"
+	"$(MK_DIR)/image-builder/image_builder.sh" -s "$(IMG_SIZE)" "$(DISTRO_ROOTFS)"
+
+initrd: rootfs
+	@echo Creating initrd image based on "$(DISTRO_ROOTFS)"
+	"$(MK_DIR)/initrd-builder/initrd_builder.sh" "$(DISTRO_ROOTFS)"

README.md

@@ -30,6 +30,14 @@ This section describes the terms used for all documentation in this repository.
 
   See [the image builder documentation](image-builder/README.md).
 
+- initrd (or "initramfs")
+
+  A compressed cpio archive loaded into memory and used as part of the Linux
+  startup process. During startup, the kernel unpacks it into a special
+  instance of a tmpfs that becomes the initial root file system.
+
+  See [the initrd builder documentation](initrd-builder/README.md).
+
 - "Base OS"
 
   A particular version of a Linux distribution used to create a Guest OS from.

image-builder/image_builder.sh

@@ -16,6 +16,7 @@ fi
 SCRIPT_NAME="${0##*/}"
 IMAGE="${IMAGE:-kata-containers.img}"
 AGENT_BIN=${AGENT_BIN:-kata-agent}
+AGENT_INIT=${AGENT_INIT:-no}
 
 die()
 {
@@ -58,7 +59,8 @@ Options:
 	-s Image size in MB ENV: IMG_SIZE
 
 Extra environment variables:
-	AGENT_BIN:  use it to change the expected agent binary name"
+	AGENT_BIN:  use it to change the expected agent binary name
+	AGENT_INIT: use kata agent as init process
 	USE_DOCKER: If set will build image in a Docker Container (requries docker)
 	            DEFAULT: not set
 EOT
@@ -122,6 +124,7 @@ if [ -n "${USE_DOCKER}" ] ; then
 		--runtime runc  \
 		--privileged \
 		--env IMG_SIZE="${IMG_SIZE}" \
+		--env AGENT_INIT=${AGENT_INIT} \
 		-v /dev:/dev \
 		-v "${script_dir}":"/osbuilder" \
 		-v "${ROOTFS}":"/rootfs" \
@@ -135,7 +138,7 @@ fi
 init="${ROOTFS}/sbin/init"
 [ -x "${init}" ] || [ -L ${init} ] || die "/sbin/init is not installed in ${ROOTFS_DIR}"
 OK "init is installed"
-[ -x "${ROOTFS}/bin/${AGENT_BIN}" ] || \
+[ "${AGENT_INIT}" == "yes" ] || [ -x "${ROOTFS}/bin/${AGENT_BIN}" ] || \
 	die "/bin/${AGENT_BIN} is not installed in ${ROOTFS}
 	use AGENT_BIN env variable to change the expected agent binary name"
 OK "Agent installed"

initrd-builder/README.md

@@ -0,0 +1,25 @@
+* [Creating a guest OS initrd image](#creating-a-guest-os-initrd-image)
+* [Further information](#further-information)
+
+# Kata Containers initrd image generation
+
+A Kata Containers initrd image is generated using the `initrd_builder.sh` script.
+This script uses a rootfs directory created by the `rootfs-builder/rootfs.sh` script.
+
+## Creating a guest OS initrd image
+
+To create a guest OS initrd image run:
+
+```
+$ sudo ./initrd_builder.sh path/to/rootfs
+```
+
+The `rootfs.sh` script populates the `path/to/rootfs` directory.
+
+## Further information
+
+For more information on how to use the `initrd_builder.sh` script, run:
+
+```
+$ ./initrd_builder.sh -h
+```

initrd-builder/initrd_builder.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 HyperHQ Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+script_name="${0##*/}"
+script_dir="$(dirname $(readlink -f $0))"
+
+if [ -n "$DEBUG" ] ; then
+	set -x
+fi
+
+SCRIPT_NAME="${0##*/}"
+INITRD_IMAGE="${INITRD_IMAGE:-kata-initrd.img}"
+AGENT_BIN=${AGENT_BIN:-kata-agent}
+AGENT_INIT=${AGENT_INIT:-no}
+
+die()
+{
+	local msg="$*"
+	echo "ERROR: ${msg}" >&2
+	exit 1
+}
+
+OK()
+{
+	local msg="$*"
+	echo "[OK] ${msg}" >&2
+}
+
+info()
+{
+	local msg="$*"
+	echo "INFO: ${msg}"
+}
+
+usage()
+{
+	error="${1:-0}"
+	cat <<EOT
+Usage: ${SCRIPT_NAME} [options] <rootfs-dir>
+	This script creates a Kata Containers initrd image file based on the
+	<rootfs-dir> directory.
+
+Options:
+	-h Show help
+	-o Set the path where the generated image file is stored.
+	   DEFAULT: the path stored in the environment variable INITRD_IMAGE
+
+Extra environment variables:
+	AGENT_BIN:  use it to change the expected agent binary name
+		    DEFAULT: kata-agent
+	AGENT_INIT: use kata agent as init process
+		    DEFAULT: no
+	USE_DOCKER: If set, the image builds in a Docker Container. Setting
+		    this variable requires Docker.
+	            DEFAULT: not set
+EOT
+exit "${error}"
+}
+
+while getopts "ho:" opt
+do
+	case "$opt" in
+		h)	usage ;;
+		o)	INITRD_IMAGE="${OPTARG}" ;;
+	esac
+done
+
+shift $(( $OPTIND - 1 ))
+
+ROOTFS="$1"
+
+
+[ -n "${ROOTFS}" ] || usage
+[ -d "${ROOTFS}" ] || die "${ROOTFS} is not a directory"
+
+ROOTFS=$(readlink -f ${ROOTFS})
+IMAGE_DIR=$(dirname ${INITRD_IMAGE})
+IMAGE_DIR=$(readlink -f ${IMAGE_DIR})
+IMAGE_NAME=$(basename ${INITRD_IMAGE})
+
+# The kata rootfs image expects init to be installed
+init="${ROOTFS}/sbin/init"
+[ -x "${init}" ] || [ -L ${init} ] || die "/sbin/init is not installed in ${ROOTFS_DIR}"
+OK "init is installed"
+[ "${AGENT_INIT}" == "yes" ] || [ -x "${ROOTFS}/bin/${AGENT_BIN}" ] || \
+	die "/bin/${AGENT_BIN} is not installed in ${ROOTFS}
+	use AGENT_BIN env variable to change the expected agent binary name"
+OK "Agent is installed"
+
+[ "$(id -u)" -eq 0 ] || die "$0: must be run as root"
+
+# initramfs expects /init
+mv -f ${init} "${ROOTFS}/init"
+
+info "Creating ${IMAGE_DIR}/${IMAGE_NAME} based on rootfs at ${ROOTFS}"
+( cd "${ROOTFS}" && find . | cpio -H newc -o | gzip -9 ) > "${IMAGE_DIR}"/"${IMAGE_NAME}"

rootfs-builder/README.md

@@ -1,5 +1,6 @@
 * [Supported base OSs](#supported-base-oss)
 * [Creating a rootfs](#creating-a-rootfs)
+* [Creating a rootfs with kernel modules](#creating-a-rootfs-with-kenrel-modules)
 * [Build a rootfs using Docker*](#build-a-rootfs-using-docker*)
 * [Adding support for a new guest OS](#adding-support-for-a-new-guest-os)
     * [Create template files](#create-template-files)
@@ -38,6 +39,8 @@ The rootfs must provide at least the following components:
 
   Path: `/sbin/init` - init binary called by the kernel.
 
+When the `AGENT_INIT` environment variable is set to `yes`, use Kata agent as `/sbin/init`.
+
 ## Creating a rootfs
 
 To build a rootfs for your chosen distribution, run:
@@ -46,6 +49,15 @@ To build a rootfs for your chosen distribution, run:
 $ sudo ./rootfs.sh <distro>
 ```
 
+## Creating a rootfs with kernel modules
+
+To build a rootfs with additional kernel modules, run:
+```
+$ sudo KERNEL_MODULES_DIR=${kernel_mod_dir} ./rootfs.sh <distro>
+```
+Where `kernel_mod_dir` points to the kernel modules directory to be put under the
+`/lib/modules/` directory of the created rootfs.
+
 ## Build a rootfs using Docker*
 
 Depending on the base OS to build the rootfs guest OS, it is required some

rootfs-builder/centos/config.sh

@@ -9,7 +9,10 @@
 OS_VERSION=${OS_VERSION:-7}
 
 #Mandatory Packages that must be installed
-# systemd: An init system that will start kata-agent
 # iptables: Need by Kata agent
-# udevlib.so: Need by Kata agent
+PACKAGES="iptables"
-PACKAGES="systemd iptables"
+
+#Optional packages:
+# systemd: An init system that will start kata-agent if kata-agent
+#          itself is not configured as init process.
+[ "$AGENT_INIT" == "no" ] && PACKAGES+=" systemd" || true

rootfs-builder/centos/rootfs_lib.sh

@@ -129,6 +129,7 @@ build_rootfs() {
 
 	DNF="${PKG_MANAGER} --config=$DNF_CONF -y --installroot=${ROOTFS_DIR} --noplugins"
 	$DNF install ${EXTRA_PKGS} ${PACKAGES}
+	$DNF clean all
 
 	[ -n "${ROOTFS_DIR}" ]  && rm -r "${ROOTFS_DIR}/var/cache/centos-osbuilder"
 }

rootfs-builder/clearlinux/config.sh

@@ -5,4 +5,5 @@
 
 #Use "latest" to always pull the last Clear Linux Release
 OS_VERSION=${OS_VERSION:-latest}
-PACKAGES="systemd iptables-bin libudev0-shim"
+PACKAGES="iptables-bin libudev0-shim"
+[ "$AGENT_INIT" == "no" ] && PACKAGES+=" systemd" || true

rootfs-builder/euleros/config.sh

@@ -9,7 +9,10 @@
 OS_VERSION=${OS_VERSION:-2.2}
 
 #Mandatory Packages that must be installed
-# systemd: An init system that will start kata-agent
 # iptables: Need by Kata agent
-# udevlib.so: Need by Kata agent
+PACKAGES="iptables"
-PACKAGES="systemd iptables"
+
+#Optional packages:
+# systemd: An init system that will start kata-agent if kata-agent
+#          itself is not configured as init process.
+[ "$AGENT_INIT" == "no" ] && PACKAGES+=" systemd" || true

rootfs-builder/fedora/config.sh

@@ -5,4 +5,5 @@
 
 #Fedora version to use
 OS_VERSION=${OS_VERSION:-27}
-PACKAGES="systemd iptables"
+PACKAGES="iptables"
+[ "$AGENT_INIT" == "no" ] && PACKAGES+=" systemd" || true

rootfs-builder/rootfs.sh

@@ -12,6 +12,9 @@ ROOTFS_DIR=${ROOTFS_DIR:-${PWD}/rootfs}
 AGENT_VERSION=${AGENT_VERSION:-master}
 GO_AGENT_PKG=${GO_AGENT_PKG:-github.com/kata-containers/agent}
 AGENT_BIN=${AGENT_BIN:-kata-agent}
+AGENT_INIT=${AGENT_INIT:-no}
+KERNEL_MODULES_DIR=${KERNEL_MODULES_DIR:-""}
+
 #Load default vesions for golang and other componets
 source "${script_dir}/versions.txt"
 
@@ -46,6 +49,10 @@ GO_AGENT_PKG: Change the golang package url to get the agent source code
 AGENT_BIN   : Name of the agent binary (needed to check if agent is installed)
 USE_DOCKER: If set will build rootfs in a Docker Container (requries docker)
             DEFAULT: not set
+AGENT_INIT  : Use $(AGENT_BIN) as init process.
+            DEFAULT: no
+KERNEL_MODULES_DIR: Optional kernel modules to put into the rootfs.
+		    DEFAULT: ""
 EOT
 exit "${error}"
 }
@@ -103,6 +110,25 @@ ENV PATH=\$PATH:\$GOROOT/bin:\$GOPATH/bin
 	popd
 }
 
+setup_agent_init() {
+	agent_bin="$1"
+	init_bin="$2"
+	info "Install $agent_bin as init process"
+	mv -f "${agent_bin}" ${init_bin}
+	OK "Agent is installed as init process"
+}
+
+copy_kernel_modules() {
+	local module_dir=$1
+	local rootfs_dir=$2
+
+	[ -z "module_dir" -o -z "rootfs_dir" ] && die "module dir and rootfs dir must be specified"
+
+	info "Copy kernel modules from ${KERNEL_MODULES_DIR}"
+	mkdir -p ${rootfs_dir}/lib/modules/
+	cp -a ${KERNEL_MODULES_DIR} ${rootfs_dir}/lib/modules/
+	OK "Kernel modules copied"
+}
 
 while getopts c:hr: opt
 do
@@ -117,7 +143,12 @@ shift $(($OPTIND - 1))
 
 [ -z "$GOPATH" ] && die "GOPATH not set"
 
+[ "$AGENT_INIT" == "yes" -o "$AGENT_INIT" == "no" ] || die "AGENT_INIT($AGENT_INIT) is invalid (must be yes or no)"
+
+[ -n "${KERNEL_MODULES_DIR}" ] && [ ! -d "${KERNEL_MODULES_DIR}" ] && die "KERNEL_MODULES_DIR defined but is not an existing directory"
+
 distro="$1"
+init="${ROOTFS_DIR}/sbin/init"
 
 [ -n "${distro}" ] || usage 1
 distro_config_dir="${script_dir}/${distro}"
@@ -140,6 +171,9 @@ if [ -n "${USE_DOCKER}" ] ; then
 		--build-arg https_proxy="${https_proxy}" \
 		-t "${image_name}" "${distro_config_dir}"
 
+	# fake mapping if KERNEL_MODULES_DIR is unset
+	kernel_mod_dir=${KERNEL_MODULES_DIR:-${ROOTFS_DIR}}
+
 	#Make sure we use a compatible runtime to build rootfs
 	# In case Clear Containers Runtime is installed we dont want to hit issue:
 	#https://github.com/clearcontainers/runtime/issues/828
@@ -151,9 +185,12 @@ if [ -n "${USE_DOCKER}" ] ; then
 		--env ROOTFS_DIR="/rootfs" \
 		--env GO_AGENT_PKG="${GO_AGENT_PKG}" \
 		--env AGENT_BIN="${AGENT_BIN}" \
+		--env AGENT_INIT="${AGENT_INIT}" \
 		--env GOPATH="${GOPATH}" \
+		--env KERNEL_MODULES_DIR="${KERNEL_MODULES_DIR}" \
 		-v "${script_dir}":"/osbuilder" \
 		-v "${ROOTFS_DIR}":"/rootfs" \
+		-v "${kernel_mod_dir}":"${kernel_mod_dir}" \
 		-v "${GOPATH}":"${GOPATH}" \
 		${image_name} \
 		bash /osbuilder/rootfs.sh "${distro}"
@@ -164,10 +201,7 @@ fi
 mkdir -p ${ROOTFS_DIR}
 build_rootfs ${ROOTFS_DIR}
 
-info "Check init is installed"
+[ -n "${KERNEL_MODULES_DIR}" ] && copy_kernel_modules ${KERNEL_MODULES_DIR} ${ROOTFS_DIR}
-init="${ROOTFS_DIR}/sbin/init"
-[ -x "${init}" ] || [ -L ${init} ] || die "/sbin/init is not installed in ${ROOTFS_DIR}"
-OK "init is installed"
 
 info "Pull Agent source code"
 go get -d "${GO_AGENT_PKG}" || true
@@ -175,8 +209,14 @@ OK "Pull Agent source code"
 
 info "Build agent"
 pushd "${GOPATH}/src/${GO_AGENT_PKG}"
-make INIT=no
+make INIT=${AGENT_INIT}
-make install DESTDIR="${ROOTFS_DIR}" INIT=no
+make install DESTDIR="${ROOTFS_DIR}" INIT=${AGENT_INIT}
 popd
 [ -x "${ROOTFS_DIR}/bin/${AGENT_BIN}" ] || die "/bin/${AGENT_BIN} is not installed in ${ROOTFS_DIR}"
 OK "Agent installed"
+
+[ "${AGENT_INIT}" == "yes" ] && setup_agent_init "${ROOTFS_DIR}/bin/${AGENT_BIN}" "${init}"
+
+info "Check init is installed"
+[ -x "${init}" ] || [ -L ${init} ] || die "/sbin/init is not installed in ${ROOTFS_DIR}"
+OK "init is installed"

tests/image_creation.bats

@@ -7,6 +7,7 @@
 
 rootfs_sh="$BATS_TEST_DIRNAME/../rootfs-builder/rootfs.sh"
 image_builder_sh="$BATS_TEST_DIRNAME/../image-builder/image_builder.sh"
+initrd_builder_sh="$BATS_TEST_DIRNAME/../initrd-builder/initrd_builder.sh"
 readonly tmp_dir=$(mktemp -t -d osbuilder-test.XXXXXXX)
 #FIXME: Remove image size after https://github.com/kata-containers/osbuilder/issues/25 is fixed
 readonly image_size=400
@@ -23,27 +24,50 @@ teardown(){
 	rm -rf "${tmp_dir}"
 }
 
-function build_image()
+function build_rootfs()
 {
 	distro="$1"
 	[ -n "$distro" ]
 	local rootfs="${tmp_dir}/rootfs-osbuilder"
 	sudo -E ${rootfs_sh} -r "${rootfs}" "${distro}"
-	sudo ${image_builder_sh} -s ${image_size} -o "${tmp_dir}/image.img" "${rootfs}"
+}
+
+function build_image()
+{
+	distro="$1"
+	[ -n "$distro" ]
+	local rootfs="${tmp_dir}/rootfs-osbuilder"
+	sudo -E ${image_builder_sh} -s ${image_size} -o "${tmp_dir}/image.img" "${rootfs}"
+}
+
+function build_initrd()
+{
+	distro="$1"
+	[ -n "$distro" ]
+	local rootfs="${tmp_dir}/rootfs-osbuilder"
+	sudo -E ${initrd_builder_sh} -o "${tmp_dir}/initrd-image.img" "${rootfs}"
 }
 
 @test "Can create fedora image" {
+	build_rootfs fedora
 	build_image fedora
+	build_initrd fedora
 }
 
 @test "Can create clearlinux image" {
+	build_rootfs clearlinux
 	build_image clearlinux
+	build_initrd clearlinux
 }
 
 @test "Can create centos image" {
+	build_rootfs centos
 	build_image centos
+	build_initrd centos
 }
 
 @test "Can create euleros image" {
+	build_rootfs euleros
 	build_image euleros
+	build_initrd euleros
 }