mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-06-30 09:13:29 +00:00
Merge pull request #312 from marcov/dracut
osbuilder: add dracut build method
This commit is contained in:
commit
73cee17420
23
.ci/setup.sh
23
.ci/setup.sh
@ -4,30 +4,17 @@
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
set -e
|
||||
|
||||
cidir=$(dirname "$0")
|
||||
source "${cidir}/lib.sh"
|
||||
|
||||
#Note: If add clearlinux as supported CI use a stateless os-release file
|
||||
source /etc/os-release
|
||||
|
||||
if [ "$ID" == fedora ];then
|
||||
sudo -E dnf -y install automake yamllint coreutils moreutils bc make gcc
|
||||
elif [ "$ID" == centos ];then
|
||||
sudo -E yum -y install epel-release
|
||||
sudo -E yum -y install automake yamllint coreutils moreutils bc
|
||||
elif [ "$ID" == ubuntu ];then
|
||||
sudo apt-get -qq update
|
||||
sudo apt-get install -y -qq make automake qemu-utils python-pip coreutils moreutils bc
|
||||
sudo pip install yamllint
|
||||
else
|
||||
echo "Linux distribution not supported"
|
||||
fi
|
||||
|
||||
|
||||
clone_tests_repo
|
||||
|
||||
pushd "${tests_repo_dir}"
|
||||
.ci/setup.sh
|
||||
popd
|
||||
|
||||
bash "${cidir}/static-checks.sh"
|
||||
# yq needed to correctly parse runtime/versions.yaml
|
||||
make -C ${tests_repo_dir} install-yq
|
||||
|
1
.gitignore
vendored
1
.gitignore
vendored
@ -1 +1,2 @@
|
||||
image-builder/nsdax
|
||||
dracut/Dockerfile
|
||||
|
@ -5,7 +5,7 @@
|
||||
#
|
||||
|
||||
sudo: required
|
||||
dist: trusty
|
||||
dist: bionic
|
||||
|
||||
os:
|
||||
- linux
|
||||
|
90
Makefile
90
Makefile
@ -9,15 +9,17 @@ ROOTFS_BUILDER := $(MK_DIR)/rootfs-builder/rootfs.sh
|
||||
INITRD_BUILDER := $(MK_DIR)/initrd-builder/initrd_builder.sh
|
||||
IMAGE_BUILDER := $(MK_DIR)/image-builder/image_builder.sh
|
||||
|
||||
DISTRO := centos
|
||||
BUILD_METHOD := distro
|
||||
BUILD_METHOD_LIST := distro dracut
|
||||
AGENT_INIT ?= no
|
||||
DISTRO ?= centos
|
||||
ROOTFS_BUILD_DEST := $(PWD)
|
||||
IMAGES_BUILD_DEST := $(PWD)
|
||||
DISTRO_ROOTFS := $(ROOTFS_BUILD_DEST)/$(DISTRO)_rootfs
|
||||
ROOTFS_BUILD_DEST := $(shell pwd)
|
||||
IMAGES_BUILD_DEST := $(shell pwd)
|
||||
ROOTFS_MARKER_SUFFIX := _rootfs.done
|
||||
DISTRO_ROOTFS_MARKER := $(ROOTFS_BUILD_DEST)/.$(DISTRO)$(ROOTFS_MARKER_SUFFIX)
|
||||
DISTRO_IMAGE := $(IMAGES_BUILD_DEST)/kata-containers.img
|
||||
DISTRO_INITRD := $(IMAGES_BUILD_DEST)/kata-containers-initrd.img
|
||||
TARGET_ROOTFS := $(ROOTFS_BUILD_DEST)/$(DISTRO)_rootfs
|
||||
TARGET_ROOTFS_MARKER := $(ROOTFS_BUILD_DEST)/.$(DISTRO)$(ROOTFS_MARKER_SUFFIX)
|
||||
TARGET_IMAGE := $(IMAGES_BUILD_DEST)/kata-containers.img
|
||||
TARGET_INITRD := $(IMAGES_BUILD_DEST)/kata-containers-initrd.img
|
||||
|
||||
VERSION_FILE := ./VERSION
|
||||
VERSION := $(shell grep -v ^\# $(VERSION_FILE))
|
||||
@ -25,6 +27,34 @@ COMMIT_NO := $(shell git rev-parse HEAD 2> /dev/null || true)
|
||||
COMMIT := $(if $(shell git status --porcelain --untracked-files=no),${COMMIT_NO}-dirty,${COMMIT_NO})
|
||||
VERSION_COMMIT := $(if $(COMMIT),$(VERSION)-$(COMMIT),$(VERSION))
|
||||
|
||||
ifeq ($(filter $(BUILD_METHOD),$(BUILD_METHOD_LIST)),)
|
||||
$(error Invalid BUILD_METHOD value '$(BUILD_METHOD)'. Supported values: $(BUILD_METHOD_LIST))
|
||||
endif
|
||||
|
||||
ifeq (dracut,$(BUILD_METHOD))
|
||||
DISTRO :=
|
||||
TARGET_ROOTFS := dracut_rootfs
|
||||
TARGET_ROOTFS_MARKER := $(ROOTFS_BUILD_DEST)/.dracut$(ROOTFS_MARKER_SUFFIX)
|
||||
# dracut specific variables
|
||||
DRACUT_KVERSION :=
|
||||
DRACUT_OVERLAY_DIR := $(MK_DIR)/dracut_overlay
|
||||
DRACUT_DIR := $(MK_DIR)/dracut
|
||||
DRACUT_CONF_DIR := $(DRACUT_DIR)/dracut.conf.d
|
||||
DRACUT_OPTIONS := --no-compress --conf /dev/null --confdir $(DRACUT_CONF_DIR)
|
||||
|
||||
ifneq (,$(DRACUT_KVERSION))
|
||||
# If a kernel version is not specified, do not make systemd load modules
|
||||
# at startup
|
||||
DRACUT_KMODULES := $(shell grep "^drivers=" $(DRACUT_CONF_DIR)/10-drivers.conf | sed -E "s,^drivers=\"(.*)\"$$,\1,")
|
||||
else
|
||||
DRACUT_OPTIONS += --no-kernel
|
||||
endif
|
||||
|
||||
ifeq (,$(DRACUT_OVERLAY_DIR))
|
||||
$(error DRACUT_OVERLAY_DIR cannot be empty)
|
||||
endif
|
||||
endif
|
||||
|
||||
# Set the variable to silent logs using chronic
|
||||
OSBUILDER_USE_CHRONIC :=
|
||||
|
||||
@ -53,7 +83,17 @@ rootfs-%: $(ROOTFS_BUILD_DEST)/.%$(ROOTFS_MARKER_SUFFIX)
|
||||
.PRECIOUS: $(ROOTFS_BUILD_DEST)/.%$(ROOTFS_MARKER_SUFFIX)
|
||||
$(ROOTFS_BUILD_DEST)/.%$(ROOTFS_MARKER_SUFFIX):: rootfs-builder/%
|
||||
$(call silent_run,Creating rootfs for "$*",$(ROOTFS_BUILDER) -o $(VERSION_COMMIT) -r $(ROOTFS_BUILD_DEST)/$*_rootfs $*)
|
||||
touch $@
|
||||
@touch $@
|
||||
|
||||
# To generate a dracut rootfs, we first generate a dracut initrd and then
|
||||
# extract it in a local folder.
|
||||
# Notes:
|
||||
# - assuming a not compressed initrd.
|
||||
.PRECIOUS: $(ROOTFS_BUILD_DEST)/.dracut$(ROOTFS_MARKER_SUFFIX)
|
||||
$(ROOTFS_BUILD_DEST)/.dracut$(ROOTFS_MARKER_SUFFIX): $(TARGET_INITRD)
|
||||
mkdir -p $(TARGET_ROOTFS)
|
||||
cat $< | cpio --extract --preserve-modification-time --make-directories --directory=$(TARGET_ROOTFS)
|
||||
@touch $@
|
||||
|
||||
image-%: $(IMAGES_BUILD_DEST)/kata-containers-image-%.img
|
||||
@ # DONT remove. This is not cancellation rule.
|
||||
@ -73,19 +113,37 @@ $(IMAGES_BUILD_DEST)/kata-containers-initrd-%.img: rootfs-%
|
||||
all: image initrd
|
||||
|
||||
.PHONY: rootfs
|
||||
rootfs: $(DISTRO_ROOTFS_MARKER)
|
||||
rootfs: $(TARGET_ROOTFS_MARKER)
|
||||
|
||||
.PHONY: image
|
||||
image: $(DISTRO_IMAGE)
|
||||
image: $(TARGET_IMAGE)
|
||||
|
||||
$(TARGET_IMAGE): $(TARGET_ROOTFS_MARKER)
|
||||
$(call silent_run,Creating image based on "$(TARGET_ROOTFS)",$(IMAGE_BUILDER) -o $@ "$(TARGET_ROOTFS)")
|
||||
|
||||
$(DISTRO_IMAGE): $(DISTRO_ROOTFS_MARKER)
|
||||
$(call silent_run,Creating image based on "$(DISTRO_ROOTFS)",$(IMAGE_BUILDER) "$(DISTRO_ROOTFS)")
|
||||
|
||||
.PHONY: initrd
|
||||
initrd: $(DISTRO_INITRD)
|
||||
initrd: $(TARGET_INITRD)
|
||||
|
||||
$(DISTRO_INITRD): $(DISTRO_ROOTFS_MARKER)
|
||||
$(call silent_run,Creating initrd image based on "$(DISTRO_ROOTFS)",$(INITRD_BUILDER) "$(DISTRO_ROOTFS)")
|
||||
ifeq (distro,$(BUILD_METHOD))
|
||||
$(TARGET_INITRD): $(TARGET_ROOTFS_MARKER)
|
||||
$(call silent_run,Creating initrd image based on "$(TARGET_ROOTFS)",$(INITRD_BUILDER) "$(TARGET_ROOTFS)")
|
||||
else
|
||||
$(TARGET_INITRD): $(DRACUT_OVERLAY_DIR)
|
||||
@echo Creating initrd image based on the host OS using dracut
|
||||
dracut $(DRACUT_OPTIONS) --include $< / $@ $(DRACUT_KVERSION)
|
||||
endif
|
||||
|
||||
# Notes on overlay dir:
|
||||
# - If user specified any kernel module in the dracut conf file,
|
||||
# we need to make sure these are pre-loaded at startup using
|
||||
# systemd modules-load.d
|
||||
$(DRACUT_OVERLAY_DIR):
|
||||
mkdir -p $@
|
||||
# Modules preload
|
||||
$(ROOTFS_BUILDER) -o $(VERSION_COMMIT) -r $@
|
||||
mkdir -p $@/etc/modules-load.d
|
||||
echo $(DRACUT_KMODULES) | tr " " "\n" > $@/etc/modules-load.d/kata-modules.conf
|
||||
|
||||
.PHONY: test
|
||||
test:
|
||||
@ -140,7 +198,7 @@ install-scripts:
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -rf $(DISTRO_ROOTFS_MARKER) $(DISTRO_ROOTFS) $(DISTRO_IMAGE) $(DISTRO_INITRD)
|
||||
rm -rf $(TARGET_ROOTFS_MARKER) $(TARGET_ROOTFS) $(TARGET_IMAGE) $(TARGET_INITRD) $(DRACUT_OVERLAY_DIR)
|
||||
|
||||
# Prints the name of the variable passed as suffix to the print- target,
|
||||
# E.g., if Makefile contains:
|
||||
|
106
README.md
106
README.md
@ -2,18 +2,25 @@
|
||||
|
||||
# osbuilder
|
||||
|
||||
* [Introduction](#introduction)
|
||||
* [Terms](#terms)
|
||||
* [Usage](#usage)
|
||||
* [Rootfs creation](#rootfs-creation)
|
||||
* [osbuilder](#osbuilder)
|
||||
* [Introduction](#introduction)
|
||||
* [Terms](#terms)
|
||||
* [Building](#building)
|
||||
* [Rootfs creation](#rootfs-creation)
|
||||
* [Rootfs with systemd as init](#rootfs-with-systemd-as-init)
|
||||
* [Rootfs with the agent as init](#rootfs-with-the-agent-as-init)
|
||||
* [Image creation](#image-creation)
|
||||
* [dracut based rootfs](#dracut-based-rootfs)
|
||||
* [Image creation](#image-creation)
|
||||
* [Image with systemd as init](#image-with-systemd-as-init)
|
||||
* [Image with the agent as init](#image-with-the-agent-as-init)
|
||||
* [Initrd creation](#initrd-creation)
|
||||
* [Tests](#tests)
|
||||
* [Platform-Distro Compatibility Matrix](#platform-distro-compatibility-matrix)
|
||||
* [dracut based image](#dracut-based-image)
|
||||
* [Initrd creation](#initrd-creation)
|
||||
* [Rootfs based initrd](#rootfs-based-initrd)
|
||||
* [dracut based initrd](#dracut-based-initrd)
|
||||
* [dracut options](#dracut-options)
|
||||
* [Add kernel modules](#add-kernel-modules)
|
||||
* [Testing](#testing)
|
||||
* [Platform-Distro Compatibility Matrix](#platform-distro-compatibility-matrix)
|
||||
|
||||
## Introduction
|
||||
|
||||
@ -53,16 +60,39 @@ This section describes the terms used for all documentation in this repository.
|
||||
|
||||
A particular version of a Linux distribution used to create a rootfs from.
|
||||
|
||||
## Usage
|
||||
- dracut
|
||||
|
||||
A guest OS build method where the building host is used as the Base OS.
|
||||
For more information refer to the [dracut homepage](https://dracut.wiki.kernel.org/index.php/Main_Page).
|
||||
|
||||
## Building
|
||||
|
||||
The top-level `Makefile` contains an example of how to use the available components.
|
||||
|
||||
By default, components will run on the host system. However, some components
|
||||
Two build methods are available, `distro` and `dracut`.
|
||||
By default, the `distro` build method is used, and this creates a rootfs using
|
||||
distro specific commands (e.g.: `debootstrap` for Debian or `yum` for CentOS).
|
||||
The `dracut` build method uses the distro-agnostic tool `dracut` to obtain the same goal.
|
||||
|
||||
By default components are run on the host system. However, some components
|
||||
offer the ability to run from within Docker (for ease of setup) by setting the
|
||||
`USE_DOCKER=true` variable.
|
||||
|
||||
For more detailed information, consult the documentation for a particular component.
|
||||
|
||||
When invoking the appropriate make target as showed below, a single command is used
|
||||
to generate an initrd or an image. This is what happens in details:
|
||||
1. A rootfs is generated based on the specified target distribution.
|
||||
2. The rootfs is provisioned with Kata-specific components and configuration files.
|
||||
3. The rootfs is used as a base to generate an initrd or an image.
|
||||
|
||||
When using the dracut build method however, the build sequence is different:
|
||||
1. An overlay directory is populated with Kata-specific components.
|
||||
2. dracut is instructed to merge the overlay directory with the required host-side
|
||||
filesystem components to generate an initrd.
|
||||
3. When generating an image, the initrd is extracted to obtain the base rootfs for
|
||||
the image.
|
||||
|
||||
### Rootfs creation
|
||||
|
||||
This section shows how to build a basic rootfs using the default distribution.
|
||||
@ -81,6 +111,15 @@ $ sudo -E PATH=$PATH make USE_DOCKER=true rootfs
|
||||
$ sudo -E PATH=$PATH make USE_DOCKER=true AGENT_INIT=yes rootfs
|
||||
```
|
||||
|
||||
#### dracut based rootfs
|
||||
|
||||
> **Note**: the dracut build method does not need a rootfs as a base for an image or initrd.
|
||||
However, a rootfs can be generated by extracting the generated initrd.
|
||||
|
||||
```
|
||||
$ sudo -E PATH=$PATH make BUILD_METHOD=dracut rootfs
|
||||
```
|
||||
|
||||
### Image creation
|
||||
|
||||
This section shows how to create an image from the already-created rootfs. For
|
||||
@ -99,18 +138,55 @@ $ sudo -E PATH=$PATH make USE_DOCKER=true image
|
||||
$ sudo -E PATH=$PATH make USE_DOCKER=true AGENT_INIT=yes image
|
||||
```
|
||||
|
||||
#### dracut based image
|
||||
|
||||
> Note: the dracut build method generates an image by first building an initrd,
|
||||
and then using the rootfs extracted from it.
|
||||
|
||||
```
|
||||
$ sudo -E PATH=$PATH make BUILD_METHOD=dracut image
|
||||
```
|
||||
|
||||
### Initrd creation
|
||||
|
||||
To create an initrd from the already-created rootfs with the agent acting as the init daemon:
|
||||
#### Rootfs based initrd
|
||||
|
||||
Create an initrd from the already-created rootfs and with the agent acting as the init daemon
|
||||
using:
|
||||
|
||||
```
|
||||
$ sudo -E PATH=$PATH make AGENT_INIT=yes initrd
|
||||
```
|
||||
|
||||
#### dracut based initrd
|
||||
|
||||
Create an initrd using the dracut build method with:
|
||||
|
||||
```
|
||||
$ sudo -E PATH=$PATH make BUILD_METHOD=dracut AGENT_INIT=yes initrd
|
||||
```
|
||||
|
||||
For further details,
|
||||
see [the initrd builder documentation](initrd-builder/README.md).
|
||||
|
||||
### Tests
|
||||
### dracut options
|
||||
|
||||
#### Add kernel modules
|
||||
|
||||
If the initrd or image needs to contain kernel modules, this can be done by:
|
||||
|
||||
1. Specify the name of the modules (as reported by `modinfo MODULE-NAME`) in
|
||||
`dracut/dracut.conf.d/10-drivers.conf`. For example this file can contain:
|
||||
```
|
||||
drivers="9p 9pnet 9pnet_virtio"
|
||||
```
|
||||
2. Set the `DRACUT_KVERSION` make variable to the release name of the kernel that
|
||||
is paired with the built image or initrd, using the `uname -r` format. For example:
|
||||
```
|
||||
$ make BUILD_METHOD=dracut DRACUT_KVERSION=5.2.1-23-kata AGENT_INIT=yes initrd
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
```
|
||||
$ make test
|
||||
@ -120,6 +196,12 @@ For further details, see [the tests documentation](tests/README.md).
|
||||
|
||||
## Platform-Distro Compatibility Matrix
|
||||
|
||||
The following table illustrates what target architecture is supported for each
|
||||
of the the osbuilder distributions.
|
||||
|
||||
> Note: this table is not relevant for the dracut build method, since it supports
|
||||
any Linux distribution and architecture where dracut is available.
|
||||
|
||||
| |Alpine |CentOS |Clear Linux |Debian/Ubuntu |EulerOS |Fedora |openSUSE |
|
||||
|-- |-- |-- |-- |-- |-- |-- |-- |
|
||||
|**ARM64** |:heavy_check_mark:|:heavy_check_mark:| | |:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|
|
||||
|
14
dracut/Dockerfile.in
Normal file
14
dracut/Dockerfile.in
Normal file
@ -0,0 +1,14 @@
|
||||
#
|
||||
# Copyright (c) 2019 SUSE LLC
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from opensuse/tumbleweed
|
||||
|
||||
RUN zypper --non-interactive refresh; \
|
||||
zypper --non-interactive install --no-recommends --force-resolution cpio curl dracut gcc git-core make tar; \
|
||||
zypper --non-interactive clean --all;
|
||||
|
||||
|
||||
# This will install the proper golang to build Kata components
|
||||
@INSTALL_GO@
|
17
dracut/dracut.conf.d/00-base.conf
Normal file
17
dracut/dracut.conf.d/00-base.conf
Normal file
@ -0,0 +1,17 @@
|
||||
#
|
||||
# Copyright (c) 2019 SUSE LLC
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Main dracut config for Kata Containers
|
||||
|
||||
# do NOT combine early microcode with ramdisk
|
||||
early_microcode="no"
|
||||
# do NOT install only what's needed to boot the local host
|
||||
hostonly="no"
|
||||
# do NOT store the kernel command line arguments in the initramfs
|
||||
hostonly_cmdline="no"
|
||||
# create reproducible images
|
||||
reproducible="yes"
|
||||
# dracut modules to include (NOTE: these are NOT kernel modules)
|
||||
dracutmodules="kernel-modules udev-rules syslog systemd"
|
9
dracut/dracut.conf.d/10-drivers.conf
Normal file
9
dracut/dracut.conf.d/10-drivers.conf
Normal file
@ -0,0 +1,9 @@
|
||||
#
|
||||
# Copyright (c) 2019 SUSE LLC
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Specify a space-separated set of kernel modules to copy from the host to
|
||||
# the initramfs image. For example:
|
||||
# drivers="9p 9pnet 9pnet_virtio"
|
||||
drivers=""
|
@ -1,13 +1,17 @@
|
||||
* [Supported base OSs](#supported-base-oss)
|
||||
* [Rootfs requirements](#rootfs-requirements)
|
||||
* [Creating a rootfs](#creating-a-rootfs)
|
||||
* [Creating a rootfs with kernel modules](#creating-a-rootfs-with-kernel-modules)
|
||||
* [Build a rootfs using Docker](#build-a-rootfs-using-docker)
|
||||
* [Adding support for a new guest OS](#adding-support-for-a-new-guest-os)
|
||||
* [Create template files](#create-template-files)
|
||||
* [Modify template files](#modify-template-files)
|
||||
* [Expected rootfs directory content](#expected-rootfs-directory-content)
|
||||
* [Optional - Customise the rootfs](#optional---customise-the-rootfs)
|
||||
* [Building a Guest OS rootfs for Kata Containers](#building-a-guest-os-rootfs-for-kata-containers)
|
||||
* [Supported base OSs](#supported-base-oss)
|
||||
* [Extra features](#extra-features)
|
||||
* [Supported distributions list](#supported-distributions-list)
|
||||
* [Generate Kata specific files](#generate-kata-specific-files)
|
||||
* [Rootfs requirements](#rootfs-requirements)
|
||||
* [Creating a rootfs](#creating-a-rootfs)
|
||||
* [Creating a rootfs with kernel modules](#creating-a-rootfs-with-kernel-modules)
|
||||
* [Build a rootfs using Docker](#build-a-rootfs-using-docker)
|
||||
* [Adding support for a new guest OS](#adding-support-for-a-new-guest-os)
|
||||
* [Create template files](#create-template-files)
|
||||
* [Modify template files](#modify-template-files)
|
||||
* [Expected rootfs directory content](#expected-rootfs-directory-content)
|
||||
* [Optional - Customize the rootfs](#optional---customize-the-rootfs)
|
||||
* [Adding extra packages](#adding-extra-packages)
|
||||
* [Arbitrary rootfs changes](#arbitrary-rootfs-changes)
|
||||
|
||||
@ -21,10 +25,25 @@ The `rootfs.sh` script builds a rootfs based on a particular Linux\*
|
||||
distribution. The script supports multiple distributions and can be extended
|
||||
to add further ones.
|
||||
|
||||
To list the supported distributions, run:
|
||||
### Extra features
|
||||
|
||||
#### Supported distributions list
|
||||
|
||||
List the supported distributions by running the following:
|
||||
```
|
||||
$ ./rootfs.sh -h
|
||||
$ ./rootfs.sh -l
|
||||
```
|
||||
|
||||
#### Generate Kata specific files
|
||||
The `rootfs.sh` script can be used to populate a directory with only Kata specific files and
|
||||
components, without creating a full usable rootfs.
|
||||
This feature is used to create a rootfs based on a distribution not officially
|
||||
supported by osbuilder, and when building an image using the dracut build method.
|
||||
|
||||
To achieve this, simply invoke `rootfs.sh` without specifying a target rootfs, e.g.:
|
||||
```
|
||||
$ mkdir kata-overlay
|
||||
$ ./rootfs.sh -r "$PWD/kata-overlay"
|
||||
```
|
||||
|
||||
## Rootfs requirements
|
||||
@ -158,7 +177,7 @@ After the new directory structure is created:
|
||||
After the function `build_rootfs` is called, the script expects the
|
||||
rootfs directory to contain `/sbin/init` and `/sbin/kata-agent` binaries.
|
||||
|
||||
### Optional - Customise the rootfs
|
||||
### Optional - Customize the rootfs
|
||||
|
||||
For particular use cases developers might want to modify the guest OS.
|
||||
|
||||
|
@ -14,17 +14,12 @@ script_name="${0##*/}"
|
||||
script_dir="$(dirname $(readlink -f $0))"
|
||||
AGENT_VERSION=${AGENT_VERSION:-}
|
||||
GO_AGENT_PKG=${GO_AGENT_PKG:-github.com/kata-containers/agent}
|
||||
GO_RUNTIME_PKG=${GO_RUNTIME_PKG:-github.com/kata-containers/runtime}
|
||||
AGENT_BIN=${AGENT_BIN:-kata-agent}
|
||||
AGENT_INIT=${AGENT_INIT:-no}
|
||||
KERNEL_MODULES_DIR=${KERNEL_MODULES_DIR:-""}
|
||||
OSBUILDER_VERSION="unknown"
|
||||
DOCKER_RUNTIME=${DOCKER_RUNTIME:-runc}
|
||||
GO_VERSION="null"
|
||||
#https://github.com/kata-containers/tests/blob/master/.ci/jenkins_job_build.sh
|
||||
# Give preference to variable set by CI
|
||||
KATA_BRANCH=${branch:-}
|
||||
KATA_BRANCH=${KATA_BRANCH:-master}
|
||||
export GOPATH=${GOPATH:-${HOME}/go}
|
||||
|
||||
lib_file="${script_dir}/../scripts/lib.sh"
|
||||
@ -52,18 +47,31 @@ typeset -r CONFIG_ARCH_SH="config_${ARCH}.sh"
|
||||
# build_rootfs() function.
|
||||
typeset -r LIB_SH="rootfs_lib.sh"
|
||||
|
||||
# rootfs distro name specified by the user
|
||||
typeset distro=
|
||||
|
||||
# Absolute path to the rootfs root folder
|
||||
typeset ROOTFS_DIR
|
||||
|
||||
# Absolute path in the rootfs to the "init" executable / symlink.
|
||||
# Typically something like "${ROOTFS_DIR}/init
|
||||
typeset init=
|
||||
|
||||
#$1: Error code if want to exit different to 0
|
||||
usage()
|
||||
{
|
||||
error="${1:-0}"
|
||||
cat <<EOT
|
||||
|
||||
Usage: ${script_name} [options] <distro>
|
||||
Usage: ${script_name} [options] [DISTRO]
|
||||
|
||||
Build a rootfs based on <distro> OS, to be included in a Kata Containers
|
||||
image.
|
||||
Build and setup a rootfs directory based on DISTRO OS, used to create
|
||||
Kata Containers images or initramfs.
|
||||
|
||||
Supported <distro> values:
|
||||
When no DISTRO is provided, an existing base rootfs at ROOTFS_DIR is provisioned
|
||||
with the Kata specific components and configuration.
|
||||
|
||||
Supported DISTRO values:
|
||||
$(get_distros | tr "\n" " ")
|
||||
|
||||
Options:
|
||||
@ -75,7 +83,7 @@ Options:
|
||||
yaml description.
|
||||
-r <directory> Specify the rootfs base directory. Overrides the ROOTFS_DIR
|
||||
environment variable.
|
||||
-t Print the test configuration for <distro> and exit
|
||||
-t DISTRO Print the test configuration for DISTRO and exit
|
||||
immediately.
|
||||
|
||||
Environment Variables:
|
||||
@ -100,7 +108,7 @@ DISTRO_REPO Use host repositories to install guest packages.
|
||||
GO_AGENT_PKG URL of the Git repository hosting the agent package.
|
||||
Default value: ${GO_AGENT_PKG}
|
||||
|
||||
GRACEFUL_EXIT If set, and if the <distro> configuration specifies a
|
||||
GRACEFUL_EXIT If set, and if the DISTRO configuration specifies a
|
||||
non-empty BUILD_CAN_FAIL variable, do not return with an
|
||||
error code in case any of the build step fails.
|
||||
This is used when running CI jobs, to tolerate failures for
|
||||
@ -112,7 +120,7 @@ KERNEL_MODULES_DIR Path to a directory containing kernel modules to include in
|
||||
Default value: <empty>
|
||||
|
||||
ROOTFS_DIR Path to the directory that is populated with the rootfs.
|
||||
Default value: <${script_name} path>/rootfs-<distro-name>
|
||||
Default value: <${script_name} path>/rootfs-<DISTRO-name>
|
||||
|
||||
USE_DOCKER If set, build the rootfs inside a container (requires
|
||||
Docker).
|
||||
@ -137,7 +145,9 @@ get_distros() {
|
||||
}
|
||||
|
||||
get_test_config() {
|
||||
local distro="$1"
|
||||
local -r distro="$1"
|
||||
[ -z "$distro" ] && die "No distro name specified"
|
||||
|
||||
local config="${script_dir}/${distro}/config.sh"
|
||||
source ${config}
|
||||
|
||||
@ -176,51 +186,6 @@ docker_extra_args()
|
||||
echo "$args"
|
||||
}
|
||||
|
||||
generate_dockerfile()
|
||||
{
|
||||
dir="$1"
|
||||
|
||||
case "$(uname -m)" in
|
||||
"ppc64le")
|
||||
goarch=ppc64le
|
||||
;;
|
||||
|
||||
"aarch64")
|
||||
goarch=arm64
|
||||
;;
|
||||
"s390x")
|
||||
goarch=s390x
|
||||
;;
|
||||
|
||||
*)
|
||||
goarch=amd64
|
||||
;;
|
||||
esac
|
||||
|
||||
[ -n "$http_proxy" ] && readonly set_proxy="RUN sed -i '$ a proxy="$http_proxy"' /etc/dnf/dnf.conf /etc/yum.conf; true"
|
||||
|
||||
curlOptions=("-OL")
|
||||
[ -n "$http_proxy" ] && curlOptions+=("-x $http_proxy")
|
||||
readonly install_go="
|
||||
RUN cd /tmp ; curl ${curlOptions[@]} https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${goarch}.tar.gz
|
||||
RUN tar -C /usr/ -xzf /tmp/go${GO_VERSION}.linux-${goarch}.tar.gz
|
||||
ENV GOROOT=/usr/go
|
||||
ENV PATH=\$PATH:\$GOROOT/bin:\$GOPATH/bin
|
||||
"
|
||||
|
||||
readonly dockerfile_template="Dockerfile.in"
|
||||
[ -d "${dir}" ] || die "${dir}: not a directory"
|
||||
pushd ${dir}
|
||||
[ -f "${dockerfile_template}" ] || die "${dockerfile_template}: file not found"
|
||||
sed \
|
||||
-e "s|@GO_VERSION@|${GO_VERSION}|g" \
|
||||
-e "s|@OS_VERSION@|${OS_VERSION}|g" \
|
||||
-e "s|@INSTALL_GO@|${install_go//$'\n'/\\n}|g" \
|
||||
-e "s|@SET_PROXY@|${set_proxy}|g" \
|
||||
${dockerfile_template} > Dockerfile
|
||||
popd
|
||||
}
|
||||
|
||||
setup_agent_init()
|
||||
{
|
||||
agent_bin="$1"
|
||||
@ -261,50 +226,6 @@ error_handler()
|
||||
fi
|
||||
}
|
||||
|
||||
detect_go_version()
|
||||
{
|
||||
info "Detecting agent go version"
|
||||
typeset -r yq=$(command -v yq || command -v ${GOPATH}/bin/yq)
|
||||
[ -z "$yq" ] && die "'yq' application not found (needed to parsing minimum Go version required)"
|
||||
|
||||
local runtimeRevision=""
|
||||
|
||||
# Detect runtime revision by fetching the agent's VERSION file
|
||||
local runtime_version_url="https://raw.githubusercontent.com/kata-containers/agent/${AGENT_VERSION:-master}/VERSION"
|
||||
info "Detecting runtime version using ${runtime_version_url}"
|
||||
|
||||
if runtimeRevision="$(curl -fsSL ${runtime_version_url})"; then
|
||||
[ -n "${runtimeRevision}" ] || die "failed to get agent version"
|
||||
typeset -r runtimeVersionsURL="https://raw.githubusercontent.com/kata-containers/runtime/${runtimeRevision}/versions.yaml"
|
||||
info "Getting golang version from ${runtimeVersionsURL}"
|
||||
# This may fail if we are a kata bump.
|
||||
if GO_VERSION="$(curl -fsSL "$runtimeVersionsURL" | $yq r - "languages.golang.version")"; then
|
||||
[ "$GO_VERSION" != "null" ]
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
info "Agent version has not match with a runtime version, assumming it is a PR"
|
||||
local kata_runtime_pkg_dir="${GOPATH}/src/${GO_RUNTIME_PKG}"
|
||||
if [ ! -d "${kata_runtime_pkg_dir}" ];then
|
||||
info "There is not runtime repository in filesystem (${kata_runtime_pkg_dir})"
|
||||
local runtime_versions_url="https://raw.githubusercontent.com/kata-containers/runtime/${KATA_BRANCH}/versions.yaml"
|
||||
info "Get versions file from ${runtime_versions_url}"
|
||||
GO_VERSION="$(curl -fsSL "${runtime_versions_url}" | $yq r - "languages.golang.version")"
|
||||
if [ "$?" == "0" ] && [ "$GO_VERSION" != "null" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
return 1
|
||||
fi
|
||||
|
||||
local kata_versions_file="${kata_runtime_pkg_dir}/versions.yaml"
|
||||
info "Get Go version from ${kata_versions_file}"
|
||||
GO_VERSION="$(cat "${kata_versions_file}" | $yq r - "languages.golang.version")"
|
||||
|
||||
[ "$?" == "0" ] && [ "$GO_VERSION" != "null" ]
|
||||
}
|
||||
|
||||
# Compares two SEMVER-style versions passed as arguments, up to the MINOR version
|
||||
# number.
|
||||
# Returns a zero exit code if the version specified by the first argument is
|
||||
@ -330,229 +251,293 @@ compare_versions()
|
||||
true
|
||||
}
|
||||
|
||||
while getopts a:hlo:r:t: opt
|
||||
do
|
||||
case $opt in
|
||||
a) AGENT_VERSION="${OPTARG}" ;;
|
||||
h) usage ;;
|
||||
l) get_distros | sort && exit 0;;
|
||||
o) OSBUILDER_VERSION="${OPTARG}" ;;
|
||||
r) ROOTFS_DIR="${OPTARG}" ;;
|
||||
t) get_test_config "${OPTARG}" && exit 0;;
|
||||
esac
|
||||
done
|
||||
check_env_variables()
|
||||
{
|
||||
# Fetch the first element from GOPATH as working directory
|
||||
# as go get only works against the first item in the GOPATH
|
||||
[ -z "$GOPATH" ] && die "GOPATH not set"
|
||||
GOPATH_LOCAL="${GOPATH%%:*}"
|
||||
|
||||
shift $(($OPTIND - 1))
|
||||
[ "$AGENT_INIT" == "yes" -o "$AGENT_INIT" == "no" ] || die "AGENT_INIT($AGENT_INIT) is invalid (must be yes or no)"
|
||||
|
||||
# Fetch the first element from GOPATH as working directory
|
||||
# as go get only works against the first item in the GOPATH
|
||||
[ -z "$GOPATH" ] && die "GOPATH not set"
|
||||
GOPATH_LOCAL="${GOPATH%%:*}"
|
||||
[ -n "${KERNEL_MODULES_DIR}" ] && [ ! -d "${KERNEL_MODULES_DIR}" ] && die "KERNEL_MODULES_DIR defined but is not an existing directory"
|
||||
|
||||
[ "$AGENT_INIT" == "yes" -o "$AGENT_INIT" == "no" ] || die "AGENT_INIT($AGENT_INIT) is invalid (must be yes or no)"
|
||||
[ -n "${OSBUILDER_VERSION}" ] || die "need osbuilder version"
|
||||
}
|
||||
|
||||
[ -n "${KERNEL_MODULES_DIR}" ] && [ ! -d "${KERNEL_MODULES_DIR}" ] && die "KERNEL_MODULES_DIR defined but is not an existing directory"
|
||||
# Builds a rootfs based on the distro name provided as argument
|
||||
build_rootfs_distro()
|
||||
{
|
||||
[ -n "${distro}" ] || usage 1
|
||||
distro_config_dir="${script_dir}/${distro}"
|
||||
|
||||
[ -z "${OSBUILDER_VERSION}" ] && die "need osbuilder version"
|
||||
# Source config.sh from distro
|
||||
rootfs_config="${distro_config_dir}/${CONFIG_SH}"
|
||||
source "${rootfs_config}"
|
||||
|
||||
distro="$1"
|
||||
# Source arch-specific config file
|
||||
rootfs_arch_config="${distro_config_dir}/${CONFIG_ARCH_SH}"
|
||||
if [ -f "${rootfs_arch_config}" ]; then
|
||||
source "${rootfs_arch_config}"
|
||||
fi
|
||||
|
||||
[ -n "${distro}" ] || usage 1
|
||||
distro_config_dir="${script_dir}/${distro}"
|
||||
[ -d "${distro_config_dir}" ] || die "Not found configuration directory ${distro_config_dir}"
|
||||
|
||||
# Source config.sh from distro
|
||||
rootfs_config="${distro_config_dir}/${CONFIG_SH}"
|
||||
source "${rootfs_config}"
|
||||
if [ -z "$ROOTFS_DIR" ]; then
|
||||
ROOTFS_DIR="${script_dir}/rootfs-${OS_NAME}"
|
||||
fi
|
||||
|
||||
# Source arch-specific config file
|
||||
rootfs_arch_config="${distro_config_dir}/${CONFIG_ARCH_SH}"
|
||||
if [ -f "${rootfs_arch_config}" ]; then
|
||||
source "${rootfs_arch_config}"
|
||||
fi
|
||||
if [ -e "${distro_config_dir}/${LIB_SH}" ];then
|
||||
rootfs_lib="${distro_config_dir}/${LIB_SH}"
|
||||
info "rootfs_lib.sh file found. Loading content"
|
||||
source "${rootfs_lib}"
|
||||
fi
|
||||
|
||||
[ -d "${distro_config_dir}" ] || die "Not found configuration directory ${distro_config_dir}"
|
||||
CONFIG_DIR=${distro_config_dir}
|
||||
check_function_exist "build_rootfs"
|
||||
|
||||
if [ -z "$ROOTFS_DIR" ]; then
|
||||
ROOTFS_DIR="${script_dir}/rootfs-${OS_NAME}"
|
||||
fi
|
||||
if [ -z "$INSIDE_CONTAINER" ] ; then
|
||||
# Capture errors, but only outside of the docker container
|
||||
trap error_handler ERR
|
||||
fi
|
||||
|
||||
init="${ROOTFS_DIR}/sbin/init"
|
||||
mkdir -p ${ROOTFS_DIR}
|
||||
|
||||
if [ -e "${distro_config_dir}/${LIB_SH}" ];then
|
||||
rootfs_lib="${distro_config_dir}/${LIB_SH}"
|
||||
info "rootfs_lib.sh file found. Loading content"
|
||||
source "${rootfs_lib}"
|
||||
fi
|
||||
|
||||
CONFIG_DIR=${distro_config_dir}
|
||||
check_function_exist "build_rootfs"
|
||||
|
||||
if [ -z "$INSIDE_CONTAINER" ] ; then
|
||||
# Capture errors, but only outside of the docker container
|
||||
trap error_handler ERR
|
||||
fi
|
||||
|
||||
mkdir -p ${ROOTFS_DIR}
|
||||
|
||||
detect_go_version ||
|
||||
detect_go_version ||
|
||||
die "Could not detect the required Go version for AGENT_VERSION='${AGENT_VERSION:-master}'."
|
||||
|
||||
echo "Required Go version: $GO_VERSION"
|
||||
echo "Required Go version: $GO_VERSION"
|
||||
|
||||
if [ -z "${USE_DOCKER}" ] ; then
|
||||
#Generate an error if the local Go version is too old
|
||||
foundVersion=$(go version | sed -E "s/^.+([0-9]+\.[0-9]+\.[0-9]+).*$/\1/g")
|
||||
if [ -z "${USE_DOCKER}" ] ; then
|
||||
#Generate an error if the local Go version is too old
|
||||
foundVersion=$(go version | sed -E "s/^.+([0-9]+\.[0-9]+\.[0-9]+).*$/\1/g")
|
||||
|
||||
compare_versions "$GO_VERSION" $foundVersion || \
|
||||
die "Your Go version $foundVersion is older than the minimum expected Go version $GO_VERSION"
|
||||
else
|
||||
image_name="${distro}-rootfs-osbuilder"
|
||||
|
||||
generate_dockerfile "${distro_config_dir}"
|
||||
docker build \
|
||||
--build-arg http_proxy="${http_proxy}" \
|
||||
--build-arg https_proxy="${https_proxy}" \
|
||||
-t "${image_name}" "${distro_config_dir}"
|
||||
|
||||
# fake mapping if KERNEL_MODULES_DIR is unset
|
||||
kernel_mod_dir=${KERNEL_MODULES_DIR:-${ROOTFS_DIR}}
|
||||
|
||||
docker_run_args=""
|
||||
docker_run_args+=" --rm"
|
||||
docker_run_args+=" --runtime ${DOCKER_RUNTIME}"
|
||||
|
||||
if [ -z "${AGENT_SOURCE_BIN}" ] ; then
|
||||
docker_run_args+=" --env GO_AGENT_PKG=${GO_AGENT_PKG}"
|
||||
compare_versions "$GO_VERSION" $foundVersion || \
|
||||
die "Your Go version $foundVersion is older than the minimum expected Go version $GO_VERSION"
|
||||
else
|
||||
docker_run_args+=" --env AGENT_SOURCE_BIN=${AGENT_SOURCE_BIN}"
|
||||
docker_run_args+=" -v ${AGENT_SOURCE_BIN}:${AGENT_SOURCE_BIN}"
|
||||
image_name="${distro}-rootfs-osbuilder"
|
||||
|
||||
generate_dockerfile "${distro_config_dir}"
|
||||
docker build \
|
||||
--build-arg http_proxy="${http_proxy}" \
|
||||
--build-arg https_proxy="${https_proxy}" \
|
||||
-t "${image_name}" "${distro_config_dir}"
|
||||
|
||||
# fake mapping if KERNEL_MODULES_DIR is unset
|
||||
kernel_mod_dir=${KERNEL_MODULES_DIR:-${ROOTFS_DIR}}
|
||||
|
||||
docker_run_args=""
|
||||
docker_run_args+=" --rm"
|
||||
docker_run_args+=" --runtime ${DOCKER_RUNTIME}"
|
||||
|
||||
if [ -z "${AGENT_SOURCE_BIN}" ] ; then
|
||||
docker_run_args+=" --env GO_AGENT_PKG=${GO_AGENT_PKG}"
|
||||
else
|
||||
docker_run_args+=" --env AGENT_SOURCE_BIN=${AGENT_SOURCE_BIN}"
|
||||
docker_run_args+=" -v ${AGENT_SOURCE_BIN}:${AGENT_SOURCE_BIN}"
|
||||
fi
|
||||
|
||||
docker_run_args+=" $(docker_extra_args $distro)"
|
||||
|
||||
# Relabel volumes so SELinux allows access (see docker-run(1))
|
||||
if command -v selinuxenabled > /dev/null && selinuxenabled ; then
|
||||
for volume_dir in "${script_dir}" \
|
||||
"${ROOTFS_DIR}" \
|
||||
"${script_dir}/../scripts" \
|
||||
"${kernel_mod_dir}" \
|
||||
"${GOPATH_LOCAL}"; do
|
||||
chcon -Rt svirt_sandbox_file_t "$volume_dir"
|
||||
done
|
||||
fi
|
||||
|
||||
#Make sure we use a compatible runtime to build rootfs
|
||||
# In case Clear Containers Runtime is installed we dont want to hit issue:
|
||||
#https://github.com/clearcontainers/runtime/issues/828
|
||||
docker run \
|
||||
--env https_proxy="${https_proxy}" \
|
||||
--env http_proxy="${http_proxy}" \
|
||||
--env AGENT_VERSION="${AGENT_VERSION}" \
|
||||
--env ROOTFS_DIR="/rootfs" \
|
||||
--env AGENT_BIN="${AGENT_BIN}" \
|
||||
--env AGENT_INIT="${AGENT_INIT}" \
|
||||
--env GOPATH="${GOPATH_LOCAL}" \
|
||||
--env KERNEL_MODULES_DIR="${KERNEL_MODULES_DIR}" \
|
||||
--env EXTRA_PKGS="${EXTRA_PKGS}" \
|
||||
--env OSBUILDER_VERSION="${OSBUILDER_VERSION}" \
|
||||
--env INSIDE_CONTAINER=1 \
|
||||
--env SECCOMP="${SECCOMP}" \
|
||||
--env DEBUG="${DEBUG}" \
|
||||
-v "${script_dir}":"/osbuilder" \
|
||||
-v "${ROOTFS_DIR}":"/rootfs" \
|
||||
-v "${script_dir}/../scripts":"/scripts" \
|
||||
-v "${kernel_mod_dir}":"${kernel_mod_dir}" \
|
||||
-v "${GOPATH_LOCAL}":"${GOPATH_LOCAL}" \
|
||||
$docker_run_args \
|
||||
${image_name} \
|
||||
bash /osbuilder/rootfs.sh "${distro}"
|
||||
|
||||
exit $?
|
||||
fi
|
||||
|
||||
docker_run_args+=" $(docker_extra_args $distro)"
|
||||
build_rootfs ${ROOTFS_DIR}
|
||||
}
|
||||
|
||||
# Relabel volumes so SELinux allows access (see docker-run(1))
|
||||
if command -v selinuxenabled > /dev/null && selinuxenabled ; then
|
||||
for volume_dir in "${script_dir}" \
|
||||
"${ROOTFS_DIR}" \
|
||||
"${script_dir}/../scripts" \
|
||||
"${kernel_mod_dir}" \
|
||||
"${GOPATH_LOCAL}"; do
|
||||
chcon -Rt svirt_sandbox_file_t "$volume_dir"
|
||||
done
|
||||
# Used to create a minimal directory tree where the agent can be instaleld.
|
||||
# This is used when a distro is not specified.
|
||||
prepare_overlay()
|
||||
{
|
||||
pushd "${ROOTFS_DIR}" > /dev/null
|
||||
mkdir -p ./etc ./lib/systemd ./sbin ./var
|
||||
ln -sf ./usr/lib/systemd/systemd ./init
|
||||
ln -sf ../../init ./lib/systemd/systemd
|
||||
ln -sf ../init ./sbin/init
|
||||
# Kata sytemd unit file
|
||||
mkdir -p ./etc/systemd/system/basic.target.wants/
|
||||
ln -sf /usr/lib/systemd/system/kata-containers.target ./etc/systemd/system/basic.target.wants/kata-containers.target
|
||||
popd > /dev/null
|
||||
}
|
||||
|
||||
# Setup an existing rootfs directory, based on the OPTIONAL distro name
|
||||
# provided as argument
|
||||
setup_rootfs()
|
||||
{
|
||||
[ -z "$distro" ] && prepare_overlay
|
||||
|
||||
info "Create symlink to /tmp in /var to create private temporal directories with systemd"
|
||||
pushd "${ROOTFS_DIR}" >> /dev/null
|
||||
if [ "$PWD" != "/" ] ; then
|
||||
rm -rf ./var/cache/ ./var/lib ./var/log ./var/tmp
|
||||
fi
|
||||
|
||||
#Make sure we use a compatible runtime to build rootfs
|
||||
# In case Clear Containers Runtime is installed we dont want to hit issue:
|
||||
#https://github.com/clearcontainers/runtime/issues/828
|
||||
docker run \
|
||||
--env https_proxy="${https_proxy}" \
|
||||
--env http_proxy="${http_proxy}" \
|
||||
--env AGENT_VERSION="${AGENT_VERSION}" \
|
||||
--env ROOTFS_DIR="/rootfs" \
|
||||
--env AGENT_BIN="${AGENT_BIN}" \
|
||||
--env AGENT_INIT="${AGENT_INIT}" \
|
||||
--env GOPATH="${GOPATH_LOCAL}" \
|
||||
--env KERNEL_MODULES_DIR="${KERNEL_MODULES_DIR}" \
|
||||
--env EXTRA_PKGS="${EXTRA_PKGS}" \
|
||||
--env OSBUILDER_VERSION="${OSBUILDER_VERSION}" \
|
||||
--env INSIDE_CONTAINER=1 \
|
||||
--env SECCOMP="${SECCOMP}" \
|
||||
--env DEBUG="${DEBUG}" \
|
||||
-v "${script_dir}":"/osbuilder" \
|
||||
-v "${ROOTFS_DIR}":"/rootfs" \
|
||||
-v "${script_dir}/../scripts":"/scripts" \
|
||||
-v "${kernel_mod_dir}":"${kernel_mod_dir}" \
|
||||
-v "${GOPATH_LOCAL}":"${GOPATH_LOCAL}" \
|
||||
$docker_run_args \
|
||||
${image_name} \
|
||||
bash /osbuilder/rootfs.sh "${distro}"
|
||||
ln -s ../tmp ./var/
|
||||
|
||||
exit $?
|
||||
fi
|
||||
# For some distros tmp.mount may not be installed by default in systemd paths
|
||||
if ! [ -f "./etc/systemd/system/tmp.mount" ] && \
|
||||
! [ -f "./usr/lib/systemd/system/tmp.mount" ] &&
|
||||
[ "$AGENT_INIT" != "yes" ]; then
|
||||
local unitFile="./etc/systemd/system/tmp.mount"
|
||||
info "Install tmp.mount in ./etc/systemd/system"
|
||||
mkdir -p `dirname "$unitFile"`
|
||||
cp ./usr/share/systemd/tmp.mount "$unitFile" || cat > "$unitFile" << EOT
|
||||
# This file is part of systemd.
|
||||
#
|
||||
# systemd is free software; you can redistribute it and/or modify it
|
||||
# under the terms of the GNU Lesser General Public License as published by
|
||||
# the Free Software Foundation; either version 2.1 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
build_rootfs ${ROOTFS_DIR}
|
||||
pushd "${ROOTFS_DIR}" >> /dev/null
|
||||
if [ "$PWD" != "/" ] ; then
|
||||
rm -rf ./var/cache/ ./var/lib ./var/log
|
||||
fi
|
||||
[Unit]
|
||||
Description=Temporary Directory (/tmp)
|
||||
Documentation=man:hier(7)
|
||||
Documentation=https://www.freedesktop.org/wiki/Software/systemd/APIFileSystems
|
||||
ConditionPathIsSymbolicLink=!/tmp
|
||||
DefaultDependencies=no
|
||||
Conflicts=umount.target
|
||||
Before=local-fs.target umount.target
|
||||
After=swap.target
|
||||
|
||||
info "Create symlink to /tmp in /var to create private temporal directories with systemd"
|
||||
rm -rf ./var/tmp
|
||||
ln -s ../tmp ./var/
|
||||
[Mount]
|
||||
What=tmpfs
|
||||
Where=/tmp
|
||||
Type=tmpfs
|
||||
Options=mode=1777,strictatime,nosuid,nodev
|
||||
EOT
|
||||
fi
|
||||
|
||||
# For some distros tmp.mount may not be installed by default in systemd paths
|
||||
if ! [ -f "./etc/systemd/system/tmp.mount" ] && \
|
||||
! [ -f "./usr/lib/systemd/system/tmp.mount" ] &&
|
||||
[ "$AGENT_INIT" != "yes" ]; then
|
||||
info "Install tmp.mount in ./etc/systemd/system"
|
||||
cp ./usr/share/systemd/tmp.mount ./etc/systemd/system/tmp.mount
|
||||
fi
|
||||
popd >> /dev/null
|
||||
|
||||
popd >> /dev/null
|
||||
[ -n "${KERNEL_MODULES_DIR}" ] && copy_kernel_modules ${KERNEL_MODULES_DIR} ${ROOTFS_DIR}
|
||||
|
||||
[ -n "${KERNEL_MODULES_DIR}" ] && copy_kernel_modules ${KERNEL_MODULES_DIR} ${ROOTFS_DIR}
|
||||
chrony_conf_file="${ROOTFS_DIR}/etc/chrony.conf"
|
||||
if [ "${distro}" == "ubuntu" ] || [ "${distro}" == "debian" ] ; then
|
||||
chrony_conf_file="${ROOTFS_DIR}/etc/chrony/chrony.conf"
|
||||
fi
|
||||
|
||||
chrony_conf_file="${ROOTFS_DIR}/etc/chrony.conf"
|
||||
if [ ${distro} == ubuntu ] || [ ${distro} == debian ] ; then
|
||||
chrony_conf_file="${ROOTFS_DIR}/etc/chrony/chrony.conf"
|
||||
fi
|
||||
info "Create ${ROOTFS_DIR}/etc"
|
||||
mkdir -p "${ROOTFS_DIR}/etc"
|
||||
|
||||
info "Create ${ROOTFS_DIR}/etc"
|
||||
mkdir -p "${ROOTFS_DIR}/etc"
|
||||
|
||||
info "Configure chrony file ${chrony_conf_file}"
|
||||
cat >> "${chrony_conf_file}" <<EOT
|
||||
info "Configure chrony file ${chrony_conf_file}"
|
||||
cat >> "${chrony_conf_file}" <<EOT
|
||||
refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0
|
||||
# Step the system clock instead of slewing it if the adjustment is larger than
|
||||
# one second, at any time
|
||||
makestep 1 -1
|
||||
EOT
|
||||
|
||||
# Comment out ntp sources for chrony to be extra careful
|
||||
# Reference: https://chrony.tuxfamily.org/doc/3.4/chrony.conf.html
|
||||
sed -i 's/^\(server \|pool \|peer \)/# &/g' ${chrony_conf_file}
|
||||
# Comment out ntp sources for chrony to be extra careful
|
||||
# Reference: https://chrony.tuxfamily.org/doc/3.4/chrony.conf.html
|
||||
sed -i 's/^\(server \|pool \|peer \)/# &/g' ${chrony_conf_file}
|
||||
|
||||
chrony_systemd_service="${ROOTFS_DIR}/usr/lib/systemd/system/chronyd.service"
|
||||
if [ ${distro} == ubuntu ] || [ ${distro} == debian ] ; then
|
||||
chrony_systemd_service="${ROOTFS_DIR}/lib/systemd/system/chrony.service"
|
||||
fi
|
||||
# The CC on s390x for fedora needs to be manually set to gcc when the golang is downloaded from the main page.
|
||||
# See issue: https://github.com/kata-containers/osbuilder/issues/217
|
||||
[ "$distro" == "fedora" ] && [ "$ARCH" == "s390x" ] && export CC=gcc
|
||||
|
||||
if [ -f "$chrony_systemd_service" ]; then
|
||||
sed -i '/^\[Unit\]/a ConditionPathExists=\/dev\/ptp0' ${chrony_systemd_service}
|
||||
fi
|
||||
AGENT_DIR="${ROOTFS_DIR}/usr/bin"
|
||||
AGENT_DEST="${AGENT_DIR}/${AGENT_BIN}"
|
||||
|
||||
# The CC on s390x for fedora needs to be manually set to gcc when the golang is downloaded from the main page.
|
||||
# See issue: https://github.com/kata-containers/osbuilder/issues/217
|
||||
[ "$distro" == fedora ] && [ "$ARCH" == "s390x" ] && export CC=gcc
|
||||
if [ -z "${AGENT_SOURCE_BIN}" ] ; then
|
||||
info "Pull Agent source code"
|
||||
go get -d "${GO_AGENT_PKG}" || true
|
||||
OK "Pull Agent source code"
|
||||
|
||||
AGENT_DIR="${ROOTFS_DIR}/usr/bin"
|
||||
AGENT_DEST="${AGENT_DIR}/${AGENT_BIN}"
|
||||
info "Build agent"
|
||||
pushd "${GOPATH_LOCAL}/src/${GO_AGENT_PKG}"
|
||||
[ -n "${AGENT_VERSION}" ] && git checkout "${AGENT_VERSION}" && OK "git checkout successful"
|
||||
make clean
|
||||
make INIT=${AGENT_INIT}
|
||||
make install DESTDIR="${ROOTFS_DIR}" INIT=${AGENT_INIT} SECCOMP=${SECCOMP}
|
||||
popd
|
||||
else
|
||||
cp ${AGENT_SOURCE_BIN} ${AGENT_DEST}
|
||||
OK "cp ${AGENT_SOURCE_BIN} ${AGENT_DEST}"
|
||||
fi
|
||||
|
||||
if [ -z "${AGENT_SOURCE_BIN}" ] ; then
|
||||
info "Pull Agent source code"
|
||||
go get -d "${GO_AGENT_PKG}" || true
|
||||
OK "Pull Agent source code"
|
||||
[ -x "${AGENT_DEST}" ] || die "${AGENT_DEST} is not installed in ${ROOTFS_DIR}"
|
||||
OK "Agent installed"
|
||||
|
||||
info "Build agent"
|
||||
pushd "${GOPATH_LOCAL}/src/${GO_AGENT_PKG}"
|
||||
[ -n "${AGENT_VERSION}" ] && git checkout "${AGENT_VERSION}" && OK "git checkout successful"
|
||||
make clean
|
||||
make INIT=${AGENT_INIT}
|
||||
make install DESTDIR="${ROOTFS_DIR}" INIT=${AGENT_INIT} SECCOMP=${SECCOMP}
|
||||
popd
|
||||
else
|
||||
cp ${AGENT_SOURCE_BIN} ${AGENT_DEST}
|
||||
OK "cp ${AGENT_SOURCE_BIN} ${AGENT_DEST}"
|
||||
fi
|
||||
[ "${AGENT_INIT}" == "yes" ] && setup_agent_init "${AGENT_DEST}" "${init}"
|
||||
|
||||
[ -x "${AGENT_DEST}" ] || die "${AGENT_DEST} is not installed in ${ROOTFS_DIR}"
|
||||
OK "Agent installed"
|
||||
info "Check init is installed"
|
||||
[ -x "${init}" ] || [ -L "${init}" ] || die "/sbin/init is not installed in ${ROOTFS_DIR}"
|
||||
OK "init is installed"
|
||||
|
||||
[ "${AGENT_INIT}" == "yes" ] && setup_agent_init "${AGENT_DEST}" "${init}"
|
||||
info "Creating summary file"
|
||||
create_summary_file "${ROOTFS_DIR}"
|
||||
}
|
||||
|
||||
info "Check init is installed"
|
||||
[ -x "${init}" ] || [ -L "${init}" ] || die "/sbin/init is not installed in ${ROOTFS_DIR}"
|
||||
OK "init is installed"
|
||||
parse_arguments()
|
||||
{
|
||||
while getopts a:hlo:r:t: opt
|
||||
do
|
||||
case $opt in
|
||||
a) AGENT_VERSION="${OPTARG}" ;;
|
||||
h) usage ;;
|
||||
l) get_distros | sort && exit 0;;
|
||||
o) OSBUILDER_VERSION="${OPTARG}" ;;
|
||||
r) ROOTFS_DIR="${OPTARG}" ;;
|
||||
t) get_test_config "${OPTARG}" && exit 0;;
|
||||
*) die "Found an invalid option";;
|
||||
esac
|
||||
done
|
||||
|
||||
info "Creating summary file"
|
||||
create_summary_file "${ROOTFS_DIR}"
|
||||
shift $(($OPTIND - 1))
|
||||
distro="$1"
|
||||
}
|
||||
|
||||
main()
|
||||
{
|
||||
parse_arguments $*
|
||||
check_env_variables
|
||||
init="${ROOTFS_DIR}/sbin/init"
|
||||
|
||||
if [ -n "$distro" ]; then
|
||||
build_rootfs_distro
|
||||
else
|
||||
#Make sure ROOTFS_DIR is set correctly
|
||||
[ -d "${ROOTFS_DIR}" ] || die "Invalid rootfs directory: '$ROOTFS_DIR'"
|
||||
fi
|
||||
|
||||
setup_rootfs
|
||||
}
|
||||
|
||||
main $*
|
||||
|
100
scripts/lib.sh
100
scripts/lib.sh
@ -6,6 +6,13 @@
|
||||
|
||||
set -e
|
||||
|
||||
GO_AGENT_PKG=${GO_AGENT_PKG:-github.com/kata-containers/agent}
|
||||
GO_RUNTIME_PKG=${GO_RUNTIME_PKG:-github.com/kata-containers/runtime}
|
||||
#https://github.com/kata-containers/tests/blob/master/.ci/jenkins_job_build.sh
|
||||
# Give preference to variable set by CI
|
||||
KATA_BRANCH=${branch:-}
|
||||
KATA_BRANCH=${KATA_BRANCH:-master}
|
||||
|
||||
error()
|
||||
{
|
||||
local msg="$*"
|
||||
@ -213,3 +220,96 @@ EOT
|
||||
local rootfs_file="${file_dir}/$(basename "${file}")"
|
||||
info "Created summary file '${rootfs_file}' inside rootfs"
|
||||
}
|
||||
|
||||
# generate_dockerfile takes as only argument a path. It expects a Dockerfile.in
|
||||
# Dockerfile template to be present in that path, and will generate a usable
|
||||
# Dockerfile replacing the '@PLACEHOLDER@' in that Dockerfile
|
||||
generate_dockerfile()
|
||||
{
|
||||
dir="$1"
|
||||
[ -d "${dir}" ] || die "${dir}: not a directory"
|
||||
|
||||
case "$(uname -m)" in
|
||||
"ppc64le")
|
||||
goarch=ppc64le
|
||||
;;
|
||||
|
||||
"aarch64")
|
||||
goarch=arm64
|
||||
;;
|
||||
"s390x")
|
||||
goarch=s390x
|
||||
;;
|
||||
|
||||
*)
|
||||
goarch=amd64
|
||||
;;
|
||||
esac
|
||||
|
||||
[ -n "${http_proxy:-}" ] && readonly set_proxy="RUN sed -i '$ a proxy="${http_proxy:-}"' /etc/dnf/dnf.conf /etc/yum.conf; true"
|
||||
|
||||
curlOptions=("-OL")
|
||||
[ -n "${http_proxy:-}" ] && curlOptions+=("-x ${http_proxy:-}")
|
||||
readonly install_go="
|
||||
RUN cd /tmp ; curl ${curlOptions[@]} https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${goarch}.tar.gz
|
||||
RUN tar -C /usr/ -xzf /tmp/go${GO_VERSION}.linux-${goarch}.tar.gz
|
||||
ENV GOROOT=/usr/go
|
||||
ENV PATH=\$PATH:\$GOROOT/bin:\$GOPATH/bin
|
||||
"
|
||||
|
||||
readonly dockerfile_template="Dockerfile.in"
|
||||
pushd ${dir}
|
||||
[ -f "${dockerfile_template}" ] || die "${dockerfile_template}: file not found"
|
||||
sed \
|
||||
-e "s|@GO_VERSION@|${GO_VERSION}|g" \
|
||||
-e "s|@OS_VERSION@|${OS_VERSION:-}|g" \
|
||||
-e "s|@INSTALL_GO@|${install_go//$'\n'/\\n}|g" \
|
||||
-e "s|@SET_PROXY@|${set_proxy:-}|g" \
|
||||
${dockerfile_template} > Dockerfile
|
||||
popd
|
||||
}
|
||||
|
||||
detect_go_version()
|
||||
{
|
||||
info "Detecting agent go version"
|
||||
typeset -r yq=$(command -v yq || command -v ${GOPATH}/bin/yq)
|
||||
[ -z "$yq" ] && die "'yq' application not found (needed to parsing minimum Go version required)"
|
||||
|
||||
local runtimeRevision=""
|
||||
|
||||
# Detect runtime revision by fetching the agent's VERSION file
|
||||
local runtime_version_url="https://raw.githubusercontent.com/kata-containers/agent/${AGENT_VERSION:-master}/VERSION"
|
||||
info "Detecting runtime version using ${runtime_version_url}"
|
||||
|
||||
if runtimeRevision="$(curl -fsSL ${runtime_version_url})"; then
|
||||
[ -n "${runtimeRevision}" ] || die "failed to get agent version"
|
||||
typeset -r runtimeVersionsURL="https://raw.githubusercontent.com/kata-containers/runtime/${runtimeRevision}/versions.yaml"
|
||||
info "Getting golang version from ${runtimeVersionsURL}"
|
||||
# This may fail if we are a kata bump.
|
||||
if GO_VERSION="$(curl -fsSL "$runtimeVersionsURL" | $yq r - "languages.golang.version")"; then
|
||||
[ "$GO_VERSION" != "null" ]
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
info "Agent version has not match with a runtime version, assumming it is a PR"
|
||||
local kata_runtime_pkg_dir="${GOPATH}/src/${GO_RUNTIME_PKG}"
|
||||
if [ ! -d "${kata_runtime_pkg_dir}" ];then
|
||||
info "There is not runtime repository in filesystem (${kata_runtime_pkg_dir})"
|
||||
local runtime_versions_url="https://raw.githubusercontent.com/kata-containers/runtime/${KATA_BRANCH}/versions.yaml"
|
||||
info "Get versions file from ${runtime_versions_url}"
|
||||
GO_VERSION="$(curl -fsSL "${runtime_versions_url}" | $yq r - "languages.golang.version")"
|
||||
if [ "$?" == "0" ] && [ "$GO_VERSION" != "null" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
return 1
|
||||
fi
|
||||
|
||||
local kata_versions_file="${kata_runtime_pkg_dir}/versions.yaml"
|
||||
info "Get Go version from ${kata_versions_file}"
|
||||
GO_VERSION="$(cat "${kata_versions_file}" | $yq r - "languages.golang.version")"
|
||||
|
||||
[ "$?" == "0" ] && [ "$GO_VERSION" != "null" ]
|
||||
}
|
||||
|
||||
|
@ -25,6 +25,7 @@ readonly MACHINE_TYPE=`uname -m`
|
||||
readonly CI=${CI:-}
|
||||
readonly KATA_HYPERVISOR="${KATA_HYPERVISOR:-}"
|
||||
readonly ci_results_dir="/var/osbuilder/tests"
|
||||
readonly dracut_dir=${script_dir}/../dracut
|
||||
|
||||
# all distro tests must have this prefix
|
||||
readonly test_func_prefix="test_distro_"
|
||||
@ -35,7 +36,6 @@ readonly docker_build_runtime="runc"
|
||||
build_images=1
|
||||
build_initrds=1
|
||||
typeset -a distrosSystemd distrosAgent
|
||||
source ${test_config}
|
||||
# Hashes used to keep track of image sizes.
|
||||
# - Key: name of distro.
|
||||
# - Value: colon-separated roots and image sizes ("${rootfs_size}:${image_size}").
|
||||
@ -46,6 +46,9 @@ typeset -A built_initrds
|
||||
# not be started. Needed only after all images/initrd built successfully
|
||||
typeset -A showKataRunFailure=
|
||||
|
||||
source ${test_config}
|
||||
source "${script_dir}/../scripts/lib.sh"
|
||||
|
||||
usage()
|
||||
{
|
||||
cat <<EOT
|
||||
@ -178,7 +181,11 @@ exit_handler()
|
||||
return
|
||||
fi
|
||||
|
||||
[ -z "${showKataRunFailure}" ] && return
|
||||
if [ -z "${showKataRunFailure}" ]; then
|
||||
# Restore the default image in config file
|
||||
silent_run $mgr configure-image
|
||||
return
|
||||
fi
|
||||
|
||||
info "local runtime config:"
|
||||
cat /etc/kata-containers/configuration.toml >&2
|
||||
@ -268,7 +275,6 @@ setup()
|
||||
[ ! -d "${tests_repo_dir}" ] && git clone "https://${tests_repo}" "${tests_repo_dir}"
|
||||
|
||||
if [ -z "${KATA_DEV_MODE:-}" ]; then
|
||||
"${tests_repo_dir}/.ci/setup.sh"
|
||||
mkdir -p /etc/kata-containers/
|
||||
sudo cp -a /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers/configuration.toml
|
||||
else
|
||||
@ -414,6 +420,9 @@ call_make() {
|
||||
fi
|
||||
done
|
||||
|
||||
# Set a default make target
|
||||
[ "${#makeTargets[@]}" = "0" ] && makeTargets+=($targetType)
|
||||
|
||||
makeJobs=
|
||||
if [ -z "$CI" ]; then
|
||||
((makeJobs=$(nproc) / 2))
|
||||
@ -524,9 +533,9 @@ test_distros()
|
||||
|
||||
if [ "$KATA_HYPERVISOR" != "firecracker" ]; then
|
||||
if [ ${#distrosAgent[@]} -gt 0 ]; then
|
||||
info "building all rootfses with kata-agent as init"
|
||||
make_rootfs ${commonMakeVars[@]} AGENT_INIT=yes "${distrosAgent[@]}" &
|
||||
bgJobs+=($!)
|
||||
info "building all rootfses with kata-agent as init"
|
||||
make_rootfs ${commonMakeVars[@]} AGENT_INIT=yes "${distrosAgent[@]}" &
|
||||
bgJobs+=($!)
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -538,7 +547,7 @@ test_distros()
|
||||
local marker=$(make print-ROOTFS_MARKER_SUFFIX)
|
||||
[ -z "$marker" ] && die "Invalid rootfs marker"
|
||||
typeset -a completed=($(find ${tmp_rootfs} -name ".*${marker}" -exec basename {} \; | sed -E "s/\.(.+)${marker}/\1/"))
|
||||
for d in "${distrosSystemd[@]} ${distrosAgent[@]}"; do
|
||||
for d in "${distrosSystemd[@]}" "${distrosAgent[@]}"; do
|
||||
if [[ "${completed[@]}" =~ $d ]]; then
|
||||
info "- $d : completed"
|
||||
else
|
||||
@ -601,6 +610,47 @@ test_distros()
|
||||
show_stats
|
||||
}
|
||||
|
||||
test_dracut()
|
||||
{
|
||||
local initrd_path="${images_dir}/kata-containers-initrd-dracut.img"
|
||||
local image_path="${images_dir}/kata-containers-image-dracut.img"
|
||||
local rootfs_path="${tmp_rootfs}/dracut_rootfs"
|
||||
|
||||
detect_go_version ||
|
||||
die "Could not detect the required Go version for AGENT_VERSION='${AGENT_VERSION:-master}'."
|
||||
generate_dockerfile ${dracut_dir}
|
||||
info "Creating container for dracut"
|
||||
silent_run docker build -t dracut-test-osbuilder ${dracut_dir}
|
||||
|
||||
typeset -a dockerRunArgs=(\
|
||||
--rm \
|
||||
--runtime=runc \
|
||||
-v "${images_dir}:${images_dir}" \
|
||||
-v "${script_dir}/..":"${tmp_dir}" \
|
||||
-v "${tmp_rootfs}:${tmp_rootfs}" \
|
||||
-v /etc/localtime:/etc/localtime:ro \
|
||||
dracut-test-osbuilder \
|
||||
)
|
||||
typeset -a makeVars=(BUILD_METHOD=dracut TARGET_INITRD="${initrd_path}" TARGET_IMAGE=${image_path} TARGET_ROOTFS=${rootfs_path})
|
||||
|
||||
info "Making image for dracut inside a container"
|
||||
silent_run docker run ${dockerRunArgs[@]} make -C ${tmp_dir} ${makeVars[@]} rootfs
|
||||
make_image USE_DOCKER=1 ${makeVars[@]}
|
||||
local image_size=$(stat -c "%s" "${image_path}")
|
||||
local rootfs_size=$(get_rootfs_size "$rootfs_path")
|
||||
built_images["dracut"]="${rootfs_size}:${image_size}"
|
||||
info "Creating container for dracut"
|
||||
install_image_create_container $image_path
|
||||
|
||||
if [ "$KATA_HYPERVISOR" != "firecracker" ]; then
|
||||
info "Making initrd for dracut inside a container"
|
||||
silent_run docker run ${dockerRunArgs[@]} make -C ${tmp_dir} ${makeVars[@]} AGENT_INIT=yes clean initrd
|
||||
local initrd_size=$(stat -c "%s" "${initrd_path}")
|
||||
built_initrds["dracut"]="${rootfs_size}:${initrd_size}"
|
||||
install_initrd_create_container $initrd_path
|
||||
fi
|
||||
}
|
||||
|
||||
main()
|
||||
{
|
||||
local args=$(getopt \
|
||||
@ -648,7 +698,10 @@ main()
|
||||
trap exit_handler EXIT ERR
|
||||
setup
|
||||
|
||||
test_distros "$distro"
|
||||
# Run only if distro is not dracut
|
||||
[ "${distro:-}" != "dracut" ] && test_distros "$distro"
|
||||
# Run if distro is empty or it is dracut
|
||||
[ -z "$distro" ] || [ "$distro" = "dracut" ] && test_dracut
|
||||
|
||||
# We shouldn't really need a message like this but the CI can fail in
|
||||
# mysterious ways so make it clear!
|
||||
|
Loading…
Reference in New Issue
Block a user