rootfs-builder: Refactor rootfs.sh

This patch introduces a bash library (scripts/lib.sh) that concentrates
common functions. This also enhances future additions of other OSes, making
it more simple. Also, new variables were introduced in each distro config.sh
in order to parameterise the creation of package manager config (dnf/yum, in this case).
A fix to the resulting rootfs directory name (include OS name) is also fixed in this
patch.

Fixes #39
Fixes #34

Signed-off-by: Erick Cardona <erick.cardona.ruiz@intel.com>
This commit is contained in:
Erick Cardona
2018-02-16 17:16:33 -06:00
parent ae8a849b32
commit 52022701db
14 changed files with 142 additions and 419 deletions

View File

@@ -104,14 +104,6 @@ must be met:
The `rootfs.sh` script will check for immediate sub-directories
containing the following expected files:
- A `bash(1)` script called `rootfs_lib.sh`
This file must contain a function called `build_rootfs()`, which must
receive the path to where the rootfs is created, as its first argument.
Path: `rootfs-builder/<distro>/rootfs_lib.sh`.
- A `bash(1)` script called `config.sh`
This represents the specific configuration for `<distro>`. It must
@@ -121,6 +113,16 @@ containing the following expected files:
Path: `rootfs-builder/<distro>/config.sh`.
- (OPTIONAL) A `bash(1)` script called `rootfs_lib.sh`
This file must contain a function called `build_rootfs()`, which must
receive the path to where the rootfs is created, as its first argument.
Normally, this file is needed if a new distro with a special requirement
is needed. This function will override the `build_rootfs()` function in
`scripts/lib.sh`.
Path: `rootfs-builder/<distro>/rootfs_lib.sh`.
### Create template files
To create a directory with the expected file structure run:

View File

@@ -1,23 +1,18 @@
# This is a configuration file add extra variables to
# be used by build_rootfs() from rootfs_lib.sh the variables will be
# loaded just before call the function.
#
# Copyright (c) 2018 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
# Here there are a couple of variables you may need.
# Remove them or add more
OS_NAME="Alpine"
# alpine version
OS_VERSION=${OS_VERSION:-v3.7}
# Essential base packages
BASE_PACKAGES="alpine-base"
# Alpine mirror to use
# See a list of mirrors at http://nl.alpinelinux.org/alpine/MIRRORS.txt
MIRROR=http://dl-5.alpinelinux.org/alpine
# Default Architecture
ARCH=${ARCH:-x86_64}
# Mandatory Packages that must be installed
# - iptables: Need by Kata agent
PACKAGES="iptables"

View File

@@ -4,14 +4,6 @@
#
# SPDX-License-Identifier: Apache-2.0
check_root()
{
if [ "$(id -u)" != "0" ]; then
echo "Root is needed"
exit 1
fi
}
# - Arguments
# rootfs_dir=$1
#

View File

@@ -1,15 +1,25 @@
# This is a configuration file add extra variables to
# be used by build_rootfs() from rootfs_lib.sh the variables will be
# loaded just before call the function.
#
# Copyright (c) 2018 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
# Here there are a couple of variables you may need.
# Remove them or add more
OS_NAME="Centos"
# Centos Version
OS_VERSION=${OS_VERSION:-7}
#Mandatory Packages that must be installed
# iptables: Need by Kata agent
LOG_FILE="/var/log/yum-centos.log"
MIRROR_LIST="http://mirrorlist.centos.org/?release=${OS_VERSION}&arch=${ARCH}&repo=os&container=container"
# Aditional Repos
CENTOS_UPDATES_URL="http://mirrorlist.centos.org/?release=${OS_VERSION}&arch=${ARCH}&repo=updates&container=container"
CENTOS_EXTRAS_URL="http://mirrorlist.centos.org/?release=${OS_VERSION}&arch=${ARCH}&repo=extras&container=container"
CENTOS_PLUS_URL="http://mirrorlist.centos.org/?release=${OS_VERSION}&arch=${ARCH}&repo=centosplus&container=container"
GPG_KEY_FILE="RPM-GPG-KEY-CentOS-7"
PACKAGES="iptables"
#Optional packages:

View File

@@ -1,135 +0,0 @@
#!/bin/bash
#
# Copyright (c) 2017 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
check_program(){
type "$1" >/dev/null 2>&1
}
check_root()
{
if [ "$(id -u)" != "0" ]; then
echo "Root is needed"
exit 1
fi
}
generate_dnf_config()
{
cat > "${DNF_CONF}" << EOF
[main]
cachedir=/var/cache/centos-osbuilder
keepcache=0
debuglevel=2
logfile=/var/log/yum-centos.log
exactarch=1
obsoletes=1
gpgcheck=0
plugins=0
installonly_limit=3
#Dont use the default dnf reposdir
#this will prevent to use host repositories
reposdir=/root/mash
[base]
name=CentOS-7 - Base
mirrorlist=http://mirrorlist.centos.org/?release=7&arch=x86_64&repo=os&container=container
#baseurl=${REPO_URL}/os/x86_64/
gpgcheck=1
gpgkey=file://${CONFIG_DIR}/RPM-GPG-KEY-CentOS-7
#released updates
[updates]
name=CentOS-7 - Updates
mirrorlist=http://mirrorlist.centos.org/?release=7&arch=x86_64&repo=updates&container=container
#baseurl=${REPO_URL}/updates/x86_64/
gpgcheck=1
gpgkey=file://${CONFIG_DIR}/RPM-GPG-KEY-CentOS-7
#additional packages that may be useful
[extras]
name=CentOS-7 - Extras
mirrorlist=http://mirrorlist.centos.org/?release=7&arch=x86_64&repo=extras&container=container
#baseurl=${REPO_URL}/extras/x86_64/
gpgcheck=1
gpgkey=file://${CONFIG_DIR}/RPM-GPG-KEY-CentOS-7
#additional packages that extend functionality of existing packages
[centosplus]
name=CentOS-7 - Plus
mirrorlist=http://mirrorlist.centos.org/?release=7&arch=x86_64&repo=centosplus&container=container
#baseurl=${REPO_URL}/centosplus/x86_64/
gpgcheck=1
enabled=0
gpgkey=file://${CONFIG_DIR}/RPM-GPG-KEY-CentOS-7
EOF
}
# - Arguments
# rootfs_dir=$1
#
# - Optional environment variables
#
# EXTRA_PKGS: Variable to add extra PKGS provided by the user
#
# BIN_AGENT: Name of the Kata-Agent binary
#
# REPO_URL: URL to distribution repository ( should be configured in
# config.sh file)
#
# Any other configuration variable for a specific distro must be added
# and documented on its own config.sh
#
# - Expected result
#
# rootfs_dir populated with rootfs pkgs
# It must provide a binary in /sbin/init
build_rootfs() {
# Mandatory
local ROOTFS_DIR=$1
#Name of the Kata-Agent binary
local BIN_AGENT=${BIN_AGENT}
# In case of support EXTRA packages, use it to allow
# users add more packages to the base rootfs
local EXTRA_PKGS=${EXTRA_PKGS:-}
#In case rootfs is created usign repositories allow user to modify
# the default URL
local REPO_URL=${REPO_URL:-http://mirror.centos.org/centos/7}
#PATH where files this script is placed
#Use it to refer to files in the same directory
#Exmaple: ${CONFIG_DIR}/foo
local CONFIG_DIR=${CONFIG_DIR}
# Populate ROOTFS_DIR
# Must provide /sbin/init and /bin/${BIN_AGENT}
check_root
if [ ! -f "${DNF_CONF}" ]; then
DNF_CONF="./kata-centos-dnf.conf"
generate_dnf_config
fi
mkdir -p "${ROOTFS_DIR}"
if [ -n "${PKG_MANAGER}" ]; then
info "DNF path provided by user: ${PKG_MANAGER}"
elif check_program "dnf"; then
PKG_MANAGER="dnf"
elif check_program "yum" ; then
PKG_MANAGER="yum"
else
die "neither yum nor dnf is installed"
fi
info "Using : ${PKG_MANAGER} to pull packages from ${REPO_URL}"
DNF="${PKG_MANAGER} --config=$DNF_CONF -y --installroot=${ROOTFS_DIR} --noplugins"
$DNF install ${EXTRA_PKGS} ${PACKAGES}
$DNF clean all
[ -n "${ROOTFS_DIR}" ] && rm -r "${ROOTFS_DIR}/var/cache/centos-osbuilder"
}

View File

@@ -1,9 +1,19 @@
#
# Copyright (c) 2017 Intel Corporation
# Copyright (c) 2018 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
#Use "latest" to always pull the last Clear Linux Release
OS_NAME="Clear"
OS_VERSION=${OS_VERSION:-latest}
BASE_URL="https://download.clearlinux.org/current/${ARCH}/os/"
REPO_NAME="clear"
PACKAGES="iptables-bin libudev0-shim"
#Optional packages:
# systemd: An init system that will start kata-agent if kata-agent
# itself is not configured as init process.
[ "$AGENT_INIT" == "no" ] && PACKAGES+=" systemd" || true

View File

@@ -1,93 +0,0 @@
#!/bin/bash
#
# Copyright (c) 2017 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
set -e
check_program(){
type "$1" >/dev/null 2>&1
}
generate_dnf_config()
{
echo "WARNING: using not signed packages"
cat > "${DNF_CONF}" << EOF
[main]
cachedir=/var/cache/dnf-clear
keepcache=0
debuglevel=2
logfile=/var/log/dnf.log
exactarch=1
obsoletes=1
gpgcheck=0
plugins=0
installonly_limit=3
#Dont use the default dnf reposdir
#this will prevent to use host repositories
reposdir=/root/mash
[clear]
name=Clear
failovermethod=priority
baseurl=${REPO_URL}
enabled=1
#Clear Linux based packages security limitations
#Although the Clear Linux rootfs is constructed from rpm packages, Clear Linux
#itself is not an rpm-based Linux distribution (the software installed on a
#Clear Linux system is not managed using rpm). The rpm packages used to
#generate the rootfs are not signed, so there is no way to ensure that
#downloaded packages are trustworthy.
gpgcheck=0
EOF
}
build_rootfs()
{
# Mandatory
local ROOTFS_DIR=$1
#In case rootfs is created usig repositories allow user to modify
# the default URL
local REPO_URL=${REPO_URL:-https://download.clearlinux.org/current/x86_64/os/}
# In case of support EXTRA packages, use it to allow
# users add more packages to the base rootfs
local EXTRA_PKGS=${EXTRA_PKGS:-}
#PATH where files this script is placed
#Use it to refer to files in the same directory
#Exmaple: ${CONFIG_DIR}/foo
#local CONFIG_DIR=${CONFIG_DIR}
check_root
if [ ! -f "${DNF_CONF}" ]; then
DNF_CONF="./clear-dnf.conf"
generate_dnf_config
fi
mkdir -p "${ROOTFS_DIR}"
if [ -n "${PKG_MANAGER}" ]; then
info "DNF path provided by user: ${PKG_MANAGER}"
elif check_program "dnf"; then
PKG_MANAGER="dnf"
elif check_program "yum" ; then
PKG_MANAGER="yum"
else
die "neither yum nor dnf is installed"
fi
info "Using : ${PKG_MANAGER} to pull packages from ${REPO_URL}"
DNF="${PKG_MANAGER} --config=$DNF_CONF -y --installroot=${ROOTFS_DIR} --noplugins"
$DNF install ${EXTRA_PKGS} ${PACKAGES}
[ -n "${ROOTFS_DIR}" ] && rm -r "${ROOTFS_DIR}/var/cache/dnf-clear"
}
check_root()
{
if [ "$(id -u)" != "0" ]; then
echo "Root is needed"
exit 1
fi
}

View File

@@ -1,15 +1,11 @@
# This is a configuration file add extra variables to
# be used by build_rootfs() from rootfs_lib.sh the variables will be
# loaded just before call the function.
OS_NAME="EulerOS"
# Here there are a couple of variables you may need.
# Remove them or add more
# EulerOS Version
OS_VERSION=${OS_VERSION:-2.2}
#Mandatory Packages that must be installed
# iptables: Need by Kata agent
BASE_URL="http://developer.huawei.com/ict/site-euleros/euleros/repo/yum/${OS_VERSION}/os/${ARCH}/"
GPG_KEY_FILE="RPM-GPG-KEY-EulerOS"
PACKAGES="iptables"
#Optional packages:

View File

@@ -1,100 +0,0 @@
#!/bin/bash
#
# Copyright (c) 2018 Huawei Technologies Co., Ltd
#
# SPDX-License-Identifier: Apache-2.0
check_program(){
type "$1" >/dev/null 2>&1
}
check_root()
{
if [ "$(id -u)" != "0" ]; then
echo "Root is needed"
exit 1
fi
}
generate_yum_config()
{
cat > "${DNF_CONF}" << EOF
[main]
cachedir=/var/cache/euleros-osbuilder
keepcache=0
debuglevel=2
logfile=/var/log/yum-euleros.log
exactarch=1
[Base]
name=EulerOS-${OS_VERSION} Base
baseurl=http://developer.huawei.com/ict/site-euleros/euleros/repo/yum/${OS_VERSION}/os/x86_64/
enabled=1
gpgcheck=1
gpgkey=file://${CONFIG_DIR}/RPM-GPG-KEY-EulerOS
EOF
}
# - Arguments
# rootfs_dir=$1
#
# - Optional environment variables
#
# EXTRA_PKGS: Variable to add extra PKGS provided by the user
#
# BIN_AGENT: Name of the Kata-Agent binary
#
# REPO_URL: URL to distribution repository ( should be configured in
# config.sh file)
#
# Any other configuration variable for a specific distro must be added
# and documented on its own config.sh
#
# - Expected result
#
# rootfs_dir populated with rootfs pkgs
# It must provide a binary in /sbin/init
build_rootfs() {
# Mandatory
local ROOTFS_DIR=$1
#Name of the Kata-Agent binary
local BIN_AGENT=${BIN_AGENT}
# In case of support EXTRA packages, use it to allow
# users add more packages to the base rootfs
local EXTRA_PKGS=${EXTRA_PKGS:-}
#In case rootfs is created usign repositories allow user to modify
# the default URL
local REPO_URL=${REPO_URL:-http://developer.huawei.com/ict/site-euleros/euleros/repo/yum/2.2}
#PATH where files this script is placed
#Use it to refer to files in the same directory
#Exmaple: ${CONFIG_DIR}/foo
local CONFIG_DIR=${CONFIG_DIR}
# Populate ROOTFS_DIR
# Must provide /sbin/init and /bin/${BIN_AGENT}
check_root
if [ ! -f "{DNF_CONF}" ]; then
DNF_CONF="./kata-euleros-yum.repo"
generate_yum_config
fi
mkdir -p "${ROOTFS_DIR}"
if [ -n "${PKG_MANAGER}" ]; then
info "DNF path provided by user: ${PKG_MANAGER}"
elif check_program "yum" ; then
PKG_MANAGER="yum"
else
die "yum is not installed"
fi
info "Using : ${PKG_MANAGER} to pull packages from ${REPO_URL}"
DNF="${PKG_MANAGER} --config=$DNF_CONF -y --installroot=${ROOTFS_DIR} --noplugins"
$DNF install ${EXTRA_PKGS} ${PACKAGES}
[ -n "${ROOTFS_DIR}" ] && rm -r "${ROOTFS_DIR}/var/cache/euleros-osbuilder"
}

View File

@@ -1,9 +1,17 @@
#
# Copyright (c) 2017 Intel Corporation
# Copyright (c) 2018 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
#Fedora version to use
OS_NAME="Fedora"
OS_VERSION=${OS_VERSION:-27}
MIRROR_LIST="https://mirrors.fedoraproject.org/metalink?repo=fedora-${OS_VERSION}&arch=\$basearch"
PACKAGES="iptables"
#Optional packages:
# systemd: An init system that will start kata-agent if kata-agent
# itself is not configured as init process.
[ "$AGENT_INIT" == "no" ] && PACKAGES+=" systemd" || true

View File

@@ -1,6 +1,6 @@
#!/bin/bash
#
# Copyright (c) 2017 Intel Corporation
# Copyright (c) 2018 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
@@ -8,17 +8,22 @@ set -e
script_name="${0##*/}"
script_dir="$(dirname $(readlink -f $0))"
ROOTFS_DIR=${ROOTFS_DIR:-${PWD}/rootfs}
AGENT_VERSION=${AGENT_VERSION:-master}
GO_AGENT_PKG=${GO_AGENT_PKG:-github.com/kata-containers/agent}
AGENT_BIN=${AGENT_BIN:-kata-agent}
AGENT_INIT=${AGENT_INIT:-no}
KERNEL_MODULES_DIR=${KERNEL_MODULES_DIR:-""}
# Default architecture
ARCH=${ARCH:-"x86_64"}
#Load default vesions for golang and other componets
source "${script_dir}/versions.txt"
# Name of file that will implement build_rootfs
# config file
typeset -r CONFIG_SH="config.sh"
# Name of the extra file that could implement build_rootfs
typeset -r LIB_SH="rootfs_lib.sh"
if [ -n "$DEBUG" ] ; then
@@ -78,7 +83,7 @@ OK()
get_distros() {
cdirs=$(find "${script_dir}" -maxdepth 1 -type d)
find ${cdirs} -maxdepth 1 -name "${LIB_SH}" -printf '%H\n' | while read dir; do
find ${cdirs} -maxdepth 1 -name "${CONFIG_SH}" -printf '%H\n' | while read dir; do
basename "${dir}"
done
}
@@ -163,17 +168,32 @@ shift $(($OPTIND - 1))
[ -n "${KERNEL_MODULES_DIR}" ] && [ ! -d "${KERNEL_MODULES_DIR}" ] && die "KERNEL_MODULES_DIR defined but is not an existing directory"
distro="$1"
init="${ROOTFS_DIR}/sbin/init"
[ -n "${distro}" ] || usage 1
distro_config_dir="${script_dir}/${distro}"
[ -d "${distro_config_dir}" ] || die "Not found configuration directory ${distro_config_dir}"
rootfs_lib="${distro_config_dir}/${LIB_SH}"
source "${rootfs_lib}"
rootfs_config="${distro_config_dir}/config.sh"
# Source config.sh from distro
rootfs_config="${distro_config_dir}/${CONFIG_SH}"
source "${rootfs_config}"
lib_file="${script_dir}/../scripts/lib.sh"
info "Source $lib_file"
[ -e "$lib_file" ] && source "$lib_file" || true
[ -d "${distro_config_dir}" ] || die "Not found configuration directory ${distro_config_dir}"
if [ -z "$ROOTFS_DIR" ]; then
ROOTFS_DIR="${script_dir}/rootfs-${OS_NAME}"
fi
init="${ROOTFS_DIR}/sbin/init"
if [ -e "${distro_config_dir}/${LIB_SH}" ];then
rootfs_lib="${distro_config_dir}/${LIB_SH}"
info "rootfs_lib.sh file found. Loading content"
source "${rootfs_lib}"
fi
CONFIG_DIR=${distro_config_dir}
check_function_exist "build_rootfs"
@@ -206,6 +226,7 @@ if [ -n "${USE_DOCKER}" ] ; then
--env EXTRA_PKGS="${EXTRA_PKGS}" \
-v "${script_dir}":"/osbuilder" \
-v "${ROOTFS_DIR}":"/rootfs" \
-v "${script_dir}/../scripts":"/scripts" \
-v "${kernel_mod_dir}":"${kernel_mod_dir}" \
-v "${GOPATH}":"${GOPATH}" \
${image_name} \
@@ -235,5 +256,5 @@ OK "Agent installed"
[ "${AGENT_INIT}" == "yes" ] && setup_agent_init "${ROOTFS_DIR}/usr/bin/${AGENT_BIN}" "${init}"
info "Check init is installed"
[ -x "${init}" ] || [ -L ${init} ] || die "/sbin/init is not installed in ${ROOTFS_DIR}"
[ -x "${init}" ] || [ -L "${init}" ] || die "/sbin/init is not installed in ${ROOTFS_DIR}"
OK "init is installed"

View File

@@ -1,15 +1,8 @@
# This is a configuration file add extra variables to
# be used by build_rootfs() from rootfs_lib.sh the variables will be
# loaded just before call the function.
# loaded just before call the function. For more information see the
# rootfs-builder/README.md file.
# Here there are a couple of variables you may need.
# Remove them or add more
#Use it rootfs is based in a system has different versions
OS_VERSION=${OS_VERSION:-DEFAULT_VERSION}
#Mandatory Packages that must be installed
# systemd: An init system that will start kata-agent
# iptables: Need by Kata agent
# udevlib.so: Need by Kata agent
PACKAGES="systemd iptables udevlib.so"

View File

@@ -17,6 +17,12 @@
#
# rootfs_dir populated with rootfs pkgs
# It must provide a binary in /sbin/init
#
# Note: For some distros, the build_rootfs() function provided in scripts/lib.sh
# will suffice. If a new distro is introduced with a special requirement,
# then, a rootfs_builder/<distro>/rootfs_lib.sh file should be created
# using this template.
build_rootfs() {
# Mandatory
local ROOTFS_DIR=$1

68
rootfs-builder/fedora/rootfs_lib.sh → scripts/lib.sh Executable file → Normal file
View File

@@ -1,6 +1,6 @@
#!/bin/bash
#
# Copyright (c) 2017 Intel Corporation
# Copyright (c) 2018 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
@@ -10,37 +10,64 @@ check_program(){
type "$1" >/dev/null 2>&1
}
check_root()
{
if [ "$(id -u)" != "0" ]; then
echo "Root is needed"
exit 1
fi
}
generate_dnf_config()
{
REPO_NAME=${REPO_NAME:-"base"}
CACHE_DIR=${CACHE_DIR:-"/var/cache/dnf-${OS_NAME}"}
cat > "${DNF_CONF}" << EOF
[main]
cachedir=/var/cache/dnf/kata/
cachedir=${CACHE_DIR}
logfile=${LOG_FILE}
keepcache=0
debuglevel=2
logfile=/var/log/dnf.log
exactarch=1
obsoletes=1
gpgcheck=0
plugins=0
installonly_limit=3
#Dont use the default dnf reposdir
#this will prevent to use host repositories
reposdir=/root/mash
retries=5
[kata]
name=Fedora ${OS_VERSION} - \$basearch
failovermethod=priority
metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-${OS_VERSION}&arch=\$basearch
enabled=1
gpgcheck=0
EOF
if [ "$BASE_URL" != "" ]; then
cat >> "${DNF_CONF}" << EOF
[base]
name=${OS_NAME}-${OS_VERSION} ${REPO_NAME}
failovermethod=priority
baseurl=${BASE_URL}
enabled=1
EOF
elif [ "$MIRROR_LIST" != "" ]; then
cat >> "${DNF_CONF}" << EOF
[base]
name=${OS_NAME}-${OS_VERSION} ${REPO_NAME}
mirrorlist=${MIRROR_LIST}
enabled=1
EOF
fi
if [ "$GPG_KEY_FILE" != "" ]; then
cat >> "${DNF_CONF}" << EOF
gpgcheck=1
gpgkey=file://${CONFIG_DIR}/${GPG_KEY_FILE}
EOF
fi
}
build_rootfs()
{
# Mandatory
local ROOTFS_DIR=$1
local ROOTFS_DIR="$1"
# In case of support EXTRA packages, use it to allow
# users add more packages to the base rootfs
@@ -53,7 +80,7 @@ build_rootfs()
check_root
if [ ! -f "${DNF_CONF}" ]; then
DNF_CONF="./kata-fedora-dnf.conf"
DNF_CONF="./kata-${OS_NAME}-dnf.conf"
generate_dnf_config
fi
mkdir -p "${ROOTFS_DIR}"
@@ -70,14 +97,5 @@ build_rootfs()
DNF="${PKG_MANAGER} --config=$DNF_CONF -y --installroot=${ROOTFS_DIR} --noplugins"
$DNF install ${EXTRA_PKGS} ${PACKAGES}
[ -n "${ROOTFS_DIR}" ] && rm -r "${ROOTFS_DIR}/var/cache/dnf"
}
check_root()
{
if [ "$(id -u)" != "0" ]; then
echo "Root is needed"
exit 1
fi
[ -n "${ROOTFS_DIR}" ] && rm -r "${ROOTFS_DIR}${CACHE_DIR}"
}