diff --git a/.ci/setup.sh b/.ci/setup.sh index 1bab32bf7c..c5f966886a 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -14,12 +14,13 @@ bash "${cidir}/static-checks.sh" source /etc/os-release if [ "$ID" == fedora ];then - sudo -E dnf -y install automake bats + sudo -E dnf -y install automake bats yamllint elif [ "$ID" == ubuntu ];then #bats isn't available for Ubuntu trusty, need for travis sudo add-apt-repository -y ppa:duggan/bats sudo apt-get -qq update - sudo apt-get install -y -qq automake bats qemu-utils + sudo apt-get install -y -qq automake bats qemu-utils python-pip + sudo pip install yamllint else echo "Linux distribution not supported" fi diff --git a/Makefile b/Makefile index ccc2ea53e0..4053ebe97d 100644 --- a/Makefile +++ b/Makefile @@ -10,10 +10,16 @@ DISTRO_ROOTFS := "$(PWD)/$(DISTRO)_rootfs" IMG_SIZE=500 AGENT_INIT ?= no +VERSION_FILE := ./VERSION +VERSION := $(shell grep -v ^\# $(VERSION_FILE)) +COMMIT_NO := $(shell git rev-parse HEAD 2> /dev/null || true) +COMMIT := $(if $(shell git status --porcelain --untracked-files=no),${COMMIT_NO}-dirty,${COMMIT_NO}) +VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION)) + all: rootfs image initrd rootfs: @echo Creating rootfs based on "$(DISTRO)" - "$(MK_DIR)/rootfs-builder/rootfs.sh" -r "$(DISTRO_ROOTFS)" "$(DISTRO)" + "$(MK_DIR)/rootfs-builder/rootfs.sh" -o $(VERSION_COMMIT) -r "$(DISTRO_ROOTFS)" "$(DISTRO)" image: rootfs image-only diff --git a/VERSION b/VERSION new file mode 100644 index 0000000000..5bae440ccb --- /dev/null +++ b/VERSION @@ -0,0 +1,2 @@ +# This is the version of osbuilder. +0.0.1 diff --git a/image-builder/image_builder.sh b/image-builder/image_builder.sh index c9561754a5..c843d35ebf 100755 --- a/image-builder/image_builder.sh +++ b/image-builder/image_builder.sh @@ -6,48 +6,23 @@ set -e +[ -n "$DEBUG" ] && set -x + script_name="${0##*/}" script_dir="$(dirname $(readlink -f $0))" -if [ -n "$DEBUG" ] ; then - set -x -fi +lib_file="${script_dir}/../scripts/lib.sh" +source "$lib_file" -SCRIPT_NAME="${0##*/}" IMAGE="${IMAGE:-kata-containers.img}" AGENT_BIN=${AGENT_BIN:-kata-agent} AGENT_INIT=${AGENT_INIT:-no} -die() -{ - local msg="$*" - echo "ERROR: ${msg}" >&2 - exit 1 -} - -OK() -{ - local msg="$*" - echo "[OK] ${msg}" >&2 -} - -info() -{ - local msg="$*" - echo "INFO: ${msg}" -} - -warning() -{ - local msg="$*" - echo "WARNING: ${msg}" -} - usage() { error="${1:-0}" cat < +Usage: ${script_name} [options] This script will create a Kata Containers image file of an adequate size based on the directory. The size of the image can be also be specified manually @@ -129,6 +104,7 @@ if [ -n "${USE_DOCKER}" ] ; then --env AGENT_INIT=${AGENT_INIT} \ -v /dev:/dev \ -v "${script_dir}":"/osbuilder" \ + -v "${script_dir}/../scripts":"/scripts" \ -v "${ROOTFS}":"/rootfs" \ -v "${IMAGE_DIR}":"/image" \ ${image_name} \ @@ -138,7 +114,7 @@ if [ -n "${USE_DOCKER}" ] ; then fi # The kata rootfs image expect init and kata-agent to be installed init="${ROOTFS}/sbin/init" -[ -x "${init}" ] || [ -L ${init} ] || die "/sbin/init is not installed in ${ROOTFS_DIR}" +[ -x "${init}" ] || [ -L ${init} ] || die "/sbin/init is not installed in ${ROOTFS}" OK "init is installed" [ "${AGENT_INIT}" == "yes" ] || [ -x "${ROOTFS}/usr/bin/${AGENT_BIN}" ] || \ die "/usr/bin/${AGENT_BIN} is not installed in ${ROOTFS} diff --git a/initrd-builder/initrd_builder.sh b/initrd-builder/initrd_builder.sh index ab8383dfcb..56e3dacfe4 100755 --- a/initrd-builder/initrd_builder.sh +++ b/initrd-builder/initrd_builder.sh @@ -6,42 +6,23 @@ set -e +[ -n "$DEBUG" ] && set -x + script_name="${0##*/}" script_dir="$(dirname $(readlink -f $0))" -if [ -n "$DEBUG" ] ; then - set -x -fi +lib_file="${script_dir}/../scripts/lib.sh" +source "$lib_file" -SCRIPT_NAME="${0##*/}" INITRD_IMAGE="${INITRD_IMAGE:-kata-containers-initrd.img}" AGENT_BIN=${AGENT_BIN:-kata-agent} AGENT_INIT=${AGENT_INIT:-no} -die() -{ - local msg="$*" - echo "ERROR: ${msg}" >&2 - exit 1 -} - -OK() -{ - local msg="$*" - echo "[OK] ${msg}" >&2 -} - -info() -{ - local msg="$*" - echo "INFO: ${msg}" -} - usage() { error="${1:-0}" cat < +Usage: ${script_name} [options] This script creates a Kata Containers initrd image file based on the directory. @@ -55,9 +36,6 @@ Extra environment variables: DEFAULT: kata-agent AGENT_INIT: use kata agent as init process DEFAULT: no - USE_DOCKER: If set, the image builds in a Docker Container. Setting - this variable requires Docker. - DEFAULT: not set EOT exit "${error}" } diff --git a/rootfs-builder/clearlinux/config.sh b/rootfs-builder/clearlinux/config.sh index ae65459184..e9c3a9fe58 100644 --- a/rootfs-builder/clearlinux/config.sh +++ b/rootfs-builder/clearlinux/config.sh @@ -4,12 +4,16 @@ # SPDX-License-Identifier: Apache-2.0 OS_NAME="Clear" +REPO_NAME="clear" OS_VERSION=${OS_VERSION:-latest} -BASE_URL="https://download.clearlinux.org/current/${ARCH}/os/" +clr_url="https://download.clearlinux.org" -REPO_NAME="clear" +# resolve version +[ "${OS_VERSION}" = "latest" ] && OS_VERSION=$(curl -sL "${clr_url}/latest") + +BASE_URL="${clr_url}/releases/${OS_VERSION}/${REPO_NAME}/${ARCH}/os/" PACKAGES="iptables-bin libudev0-shim" diff --git a/rootfs-builder/rootfs.sh b/rootfs-builder/rootfs.sh index 292bb89251..f29d220db6 100755 --- a/rootfs-builder/rootfs.sh +++ b/rootfs-builder/rootfs.sh @@ -6,6 +6,8 @@ set -e +[ -n "$DEBUG" ] && set -x + script_name="${0##*/}" script_dir="$(dirname $(readlink -f $0))" AGENT_VERSION=${AGENT_VERSION:-master} @@ -14,24 +16,25 @@ AGENT_BIN=${AGENT_BIN:-kata-agent} AGENT_INIT=${AGENT_INIT:-no} KERNEL_MODULES_DIR=${KERNEL_MODULES_DIR:-""} +lib_file="${script_dir}/../scripts/lib.sh" +source "$lib_file" + # Default architecture ARCH=${ARCH:-"x86_64"} -#Load default vesions for golang and other componets +# Load default versions for golang and other componets source "${script_dir}/versions.txt" -# config file +# distro-specific config file typeset -r CONFIG_SH="config.sh" -# Name of the extra file that could implement build_rootfs +# Name of an optional distro-specific file which, if it exists, must implement the +# build_rootfs() function. typeset -r LIB_SH="rootfs_lib.sh" -if [ -n "$DEBUG" ] ; then - set -x -fi - -#$1: Error code if want to exit differnt to 0 -usage(){ +#$1: Error code if want to exit different to 0 +usage() +{ error="${1:-0}" cat <&2 - exit 1 -} - -info() -{ - msg="$*" - echo "INFO: ${msg}" >&2 -} - -OK() -{ - msg="$*" - echo "INFO: [OK] ${msg}" >&2 -} - get_distros() { cdirs=$(find "${script_dir}" -maxdepth 1 -type d) find ${cdirs} -maxdepth 1 -name "${CONFIG_SH}" -printf '%H\n' | while read dir; do @@ -88,13 +73,14 @@ get_distros() { done } - -check_function_exist() { +check_function_exist() +{ function_name="$1" [ "$(type -t ${function_name})" == "function" ] || die "${function_name} function was not defined" } -generate_dockerfile() { +generate_dockerfile() +{ dir="$1" case "$(arch)" in @@ -130,31 +116,43 @@ ENV PATH=\$PATH:\$GOROOT/bin:\$GOPATH/bin popd } -setup_agent_init() { +setup_agent_init() +{ agent_bin="$1" init_bin="$2" + + [ -z "$agent_bin" ] && die "need agent binary path" + [ -z "$init_bin" ] && die "need init bin path" + info "Install $agent_bin as init process" mv -f "${agent_bin}" ${init_bin} OK "Agent is installed as init process" } -copy_kernel_modules() { - local module_dir=$1 - local rootfs_dir=$2 +copy_kernel_modules() +{ + local module_dir="$1" + local rootfs_dir="$2" - [ -z "module_dir" -o -z "rootfs_dir" ] && die "module dir and rootfs dir must be specified" + [ -z "$module_dir" ] && die "need module directory" + [ -z "$rootfs_dir" ] && die "need rootfs directory" + + local destdir="${rootfs_dir}/lib/modules" info "Copy kernel modules from ${KERNEL_MODULES_DIR}" - mkdir -p ${rootfs_dir}/lib/modules/ - cp -a ${KERNEL_MODULES_DIR} ${rootfs_dir}/lib/modules/ + mkdir -p "${destdir}" + cp -a "${KERNEL_MODULES_DIR}" "${dest_dir}/" OK "Kernel modules copied" } -while getopts c:hr: opt +OSBUILDER_VERSION="unknown" + +while getopts c:ho:r: opt do case $opt in a) AGENT_VERSION="${OPTARG}" ;; h) usage ;; + o) OSBUILDER_VERSION="${OPTARG}" ;; r) ROOTFS_DIR="${OPTARG}" ;; esac done @@ -167,6 +165,8 @@ shift $(($OPTIND - 1)) [ -n "${KERNEL_MODULES_DIR}" ] && [ ! -d "${KERNEL_MODULES_DIR}" ] && die "KERNEL_MODULES_DIR defined but is not an existing directory" +[ -z "${OSBUILDER_VERSION}" ] && die "need osbuilder version" + distro="$1" [ -n "${distro}" ] || usage 1 @@ -176,10 +176,6 @@ distro_config_dir="${script_dir}/${distro}" rootfs_config="${distro_config_dir}/${CONFIG_SH}" source "${rootfs_config}" -lib_file="${script_dir}/../scripts/lib.sh" -info "Source $lib_file" -[ -e "$lib_file" ] && source "$lib_file" || true - [ -d "${distro_config_dir}" ] || die "Not found configuration directory ${distro_config_dir}" if [ -z "$ROOTFS_DIR" ]; then @@ -224,6 +220,7 @@ if [ -n "${USE_DOCKER}" ] ; then --env GOPATH="${GOPATH}" \ --env KERNEL_MODULES_DIR="${KERNEL_MODULES_DIR}" \ --env EXTRA_PKGS="${EXTRA_PKGS}" \ + --env OSBUILDER_VERSION="${OSBUILDER_VERSION}" \ -v "${script_dir}":"/osbuilder" \ -v "${ROOTFS_DIR}":"/rootfs" \ -v "${script_dir}/../scripts":"/scripts" \ @@ -250,11 +247,17 @@ make clean make INIT=${AGENT_INIT} make install DESTDIR="${ROOTFS_DIR}" INIT=${AGENT_INIT} popd -[ -x "${ROOTFS_DIR}/usr/bin/${AGENT_BIN}" ] || die "/usr/bin/${AGENT_BIN} is not installed in ${ROOTFS_DIR}" + +AGENT_DIR="${ROOTFS_DIR}/usr/bin" +AGENT_DEST="${AGENT_DIR}/${AGENT_BIN}" +[ -x "${AGENT_DEST}" ] || die "${AGENT_DEST} is not installed in ${ROOTFS_DIR}" OK "Agent installed" -[ "${AGENT_INIT}" == "yes" ] && setup_agent_init "${ROOTFS_DIR}/usr/bin/${AGENT_BIN}" "${init}" +[ "${AGENT_INIT}" == "yes" ] && setup_agent_init "${AGENT_DEST}" "${init}" info "Check init is installed" [ -x "${init}" ] || [ -L "${init}" ] || die "/sbin/init is not installed in ${ROOTFS_DIR}" OK "init is installed" + +info "Creating summary file" +create_summary_file "${ROOTFS_DIR}" diff --git a/scripts/lib.sh b/scripts/lib.sh index 6e02c6f00c..f323002054 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -6,7 +6,33 @@ set -e -check_program(){ +die() +{ + local msg="$*" + echo "ERROR: ${msg}" >&2 + exit 1 +} + +OK() +{ + local msg="$*" + echo "[OK] ${msg}" >&2 +} + +info() +{ + local msg="$*" + echo "INFO: ${msg}" +} + +warning() +{ + local msg="$*" + echo "WARNING: ${msg}" +} + +check_program() +{ type "$1" >/dev/null 2>&1 } @@ -36,7 +62,7 @@ reposdir=/root/mash retries=5 EOF if [ "$BASE_URL" != "" ]; then - cat >> "${DNF_CONF}" << EOF + cat >> "${DNF_CONF}" << EOF [base] name=${OS_NAME}-${OS_VERSION} ${REPO_NAME} @@ -45,7 +71,7 @@ baseurl=${BASE_URL} enabled=1 EOF elif [ "$MIRROR_LIST" != "" ]; then - cat >> "${DNF_CONF}" << EOF + cat >> "${DNF_CONF}" << EOF [base] name=${OS_NAME}-${OS_VERSION} ${REPO_NAME} @@ -55,13 +81,12 @@ EOF fi if [ "$GPG_KEY_FILE" != "" ]; then - cat >> "${DNF_CONF}" << EOF + cat >> "${DNF_CONF}" << EOF gpgcheck=1 gpgkey=file://${CONFIG_DIR}/${GPG_KEY_FILE} EOF fi - } build_rootfs() @@ -69,6 +94,8 @@ build_rootfs() # Mandatory local ROOTFS_DIR="$1" + [ -z "$ROOTFS_DIR" ] && die "need rootfs" + # In case of support EXTRA packages, use it to allow # users add more packages to the base rootfs local EXTRA_PKGS=${EXTRA_PKGS:-""} @@ -99,3 +126,65 @@ build_rootfs() [ -n "${ROOTFS_DIR}" ] && rm -r "${ROOTFS_DIR}${CACHE_DIR}" } + +# Create a YAML metadata file inside the rootfs. +# +# This provides useful information about the rootfs than can be interrogated +# once the rootfs has been converted into a image/initrd. +create_summary_file() +{ + local -r rootfs_dir="$1" + + [ -z "$rootfs_dir" ] && die "need rootfs" + + local -r file_dir="/var/lib/osbuilder" + local -r dir="${rootfs_dir}${file_dir}" + + local -r filename="osbuilder.yaml" + local file="${dir}/${filename}" + + local -r now=$(date '+%Y-%m-%dT%T.%N%zZ') + + # sanitise package list + PACKAGES=$(echo "$PACKAGES"|tr ' ' '\n'|sort -u|tr '\n' ' ') + + local -r packages=$(for pkg in ${PACKAGES}; do echo " - \"${pkg}\""; done) + + mkdir -p "$dir" + + # Semantic version of the summary file format. + # + # XXX: Increment every time the format of the summary file changes! + local -r format_version="0.0.1" + + local -r osbuilder_url="https://github.com/kata-containers/osbuilder" + + local agent="${AGENT_DEST}" + [ "$AGENT_INIT" = yes ] && agent="${init}" + + local -r agent_version=$("$agent" --version|awk '{print $NF}') + + cat >"$file"<<-EOT + --- + osbuilder: + url: "${osbuilder_url}" + version: "${OSBUILDER_VERSION}" + rootfs-creation-time: "${now}" + description: "osbuilder rootfs" + file-format-version: "${format_version}" + architecture: "${ARCH}" + base-distro: + name: "${OS_NAME}" + version: "${OS_VERSION}" + packages: +${packages} + agent: + url: "https://${GO_AGENT_PKG}" + name: "${AGENT_BIN}" + version: "${agent_version}" + agent-is-init-daemon: "${AGENT_INIT}" +EOT + + local rootfs_file="${file_dir}/$(basename "${file}")" + info "Created summary file '${rootfs_file}' inside rootfs" +} diff --git a/tests/image_creation.bats b/tests/image_creation.bats index 5ff97b8b56..724f098b94 100644 --- a/tests/image_creation.bats +++ b/tests/image_creation.bats @@ -27,7 +27,12 @@ teardown(){ function build_rootfs() { + local file="/var/lib/osbuilder/osbuilder.yaml" + local full="${tmp_rootfs}${file}" + sudo -E ${rootfs_sh} -r "${tmp_rootfs}" "${distro}" + + yamllint "${full}" } function build_image()