mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-08-27 20:18:57 +00:00
doc: Add doc and scripts for CCv0 agent PullImage
This commit add documentation and a script to help people to build, run, test and demo the CCv0 changes around PullImage on guest. It is currently limited to the Agent pullimage, but can be expanded as more code is shared. Fixes #2574 Signed-off-by: stevenhorsman <steven@uk.ibm.com>
This commit is contained in:
stevenhorsman
parent
af44b7a591
commit
76b70a7a82
@ -36,3 +36,6 @@
|
|||||||
- [How to use hotplug memory on arm64 in Kata Containers](how-to-hotplug-memory-arm64.md)
|
- [How to use hotplug memory on arm64 in Kata Containers](how-to-hotplug-memory-arm64.md)
|
||||||
- [How to setup swap devices in guest kernel](how-to-setup-swap-devices-in-guest-kernel.md)
|
- [How to setup swap devices in guest kernel](how-to-setup-swap-devices-in-guest-kernel.md)
|
||||||
- [How to run rootless vmm](how-to-run-rootless-vmm.md)
|
- [How to run rootless vmm](how-to-run-rootless-vmm.md)
|
||||||
|
|
||||||
|
## Confidential Containers
|
||||||
|
- [How to use build and test the Confidential Containers `CCv0` proof of concept](how-to-build-and-test-ccv0.md)
|
||||||
|
|||||||
490
docs/how-to/ccv0.sh
Executable file
490
docs/how-to/ccv0.sh
Executable file
@ -0,0 +1,490 @@
|
|||||||
|
#!/bin/bash -e
|
||||||
|
#
|
||||||
|
# Copyright (c) 2021 IBM Corporation
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
#
|
||||||
|
|
||||||
|
# Disclaimer: This script is work in progress for supporting the CCv0 prototype
|
||||||
|
# It shouldn't be considered supported by the Kata Containers community, or anyone else
|
||||||
|
|
||||||
|
# Based on https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md, but with elements of the tests/.ci scripts used
|
||||||
|
|
||||||
|
readonly script_name="$(basename "${BASH_SOURCE[0]}")"
|
||||||
|
|
||||||
|
# By default in Golang >= 1.16 GO111MODULE is set to "on", but not all modules support it, so overwrite to "auto"
|
||||||
|
export GO111MODULE="auto"
|
||||||
|
|
||||||
|
# Setup kata containers environments if not set - we default to use containerd
|
||||||
|
export CRI_CONTAINERD=${CRI_CONTAINERD:-"yes"}
|
||||||
|
export CRI_RUNTIME=${CRI_RUNTIME:-"containerd"}
|
||||||
|
export CRIO=${CRIO:-"no"}
|
||||||
|
export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
|
||||||
|
export KUBERNETES=${KUBERNETES:-"yes"}
|
||||||
|
export AGENT_INIT="${AGENT_INIT:-${TEST_INITRD:-no}}"
|
||||||
|
|
||||||
|
# Allow the user to overwrite the default repo and branch names if they want to build from a fork
|
||||||
|
export katacontainers_repo="${katacontainers_repo:-github.com/kata-containers/kata-containers}"
|
||||||
|
export katacontainers_branch="${katacontainers_branch:-CCv0}"
|
||||||
|
export kata_default_branch=${katacontainers_branch}
|
||||||
|
export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
|
||||||
|
export tests_branch="${tests_branch:-CCv0}"
|
||||||
|
export target_branch=${tests_branch} # kata-containers/ci/lib.sh uses target branch var to check out tests repo
|
||||||
|
|
||||||
|
# Create a bunch of common, derived values up front so we don't need to create them in all the different functions
|
||||||
|
. "$HOME/.profile"
|
||||||
|
if [ -z ${GOPATH} ]; then
|
||||||
|
export GOPATH=${HOME}/go
|
||||||
|
fi
|
||||||
|
export tests_repo_dir="${GOPATH}/src/${tests_repo}"
|
||||||
|
export katacontainers_repo_dir="${GOPATH}/src/${katacontainers_repo}"
|
||||||
|
export ROOTFS_DIR="${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder/rootfs"
|
||||||
|
|
||||||
|
debug_output() {
|
||||||
|
if [ -n "${DEBUG}" ]
|
||||||
|
then
|
||||||
|
echo "$(date): $@"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
debug_function() {
|
||||||
|
debug_output "> $@"
|
||||||
|
start=$(date +%s%N | cut -b1-13)
|
||||||
|
$@;
|
||||||
|
status=$?
|
||||||
|
end=$(date +%s%N | cut -b1-13)
|
||||||
|
time=`expr ${end} - ${start}`
|
||||||
|
debug_output "< $@. Time taken: $(echo "scale=2; ${time} / 1000" | bc -l)s. RC: ${status}"
|
||||||
|
}
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
exit_code="$1"
|
||||||
|
cat <<EOT
|
||||||
|
Overview:
|
||||||
|
Build and test kata containers from source
|
||||||
|
Optionally set kata-containers and tests repo and branch as exported variables before running
|
||||||
|
e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && . ~/${script_name} -d build_and_install_all
|
||||||
|
Usage:
|
||||||
|
${script_name} [options] <command>
|
||||||
|
Commands:
|
||||||
|
- help: Display this help
|
||||||
|
- all: Build and install everything, test kata with containerd and capture the logs
|
||||||
|
- build_and_install_all: Build and install everything
|
||||||
|
- initialize: Install dependencies and check out kata-containers source
|
||||||
|
- rebuild_and_install_kata: Rebuild the kata runtime and agent and build and install the image
|
||||||
|
- build_kata_runtime: Build and install the kata runtime
|
||||||
|
- configure: Configure Kata to use rootfs and enable debug
|
||||||
|
- create_rootfs: Create a local rootfs
|
||||||
|
- build_and_add_agent_to_rootfs:Builds the kata-agent and adds it to the rootfs
|
||||||
|
- build_and_install_rootfs: Builds and installs the rootfs image
|
||||||
|
- install_guest_kernel: Setup, build and install the guest kernel
|
||||||
|
- build_qemu: Checkout, patch, build and install QEMU
|
||||||
|
- init_kubernetes: initialize a Kubernetes cluster on this system
|
||||||
|
- create_kata_pod: Create a kata runtime nginx pod in Kubernetes
|
||||||
|
- delete_kata_pod: Delete a kata runtime nginx pod in Kubernetes
|
||||||
|
- open_kata_console: Stream the kata runtime's console
|
||||||
|
- open_kata_shell: Open a shell into the kata runtime
|
||||||
|
- agent_pull_image: Run PullImage command against the agent with agent-ctl
|
||||||
|
- agent_list_commands: List agent commands on agent-ctl
|
||||||
|
- test: Test using kata with containerd
|
||||||
|
- test_capture_logs: Test using kata with containerd and capture the logs in the user's home directory
|
||||||
|
|
||||||
|
Options:
|
||||||
|
-d: Enable debug
|
||||||
|
-h: Display this help
|
||||||
|
EOT
|
||||||
|
# if script sourced don't exit as this will exit the main shell, just return instead
|
||||||
|
[[ $_ != $0 ]] && return "$exit_code" || exit "$exit_code"
|
||||||
|
}
|
||||||
|
|
||||||
|
build_and_install_all() {
|
||||||
|
initialize
|
||||||
|
build_and_install_kata_runtime
|
||||||
|
configure
|
||||||
|
create_a_local_rootfs
|
||||||
|
build_and_install_rootfs
|
||||||
|
install_guest_kernel_image
|
||||||
|
build_qemu
|
||||||
|
build_bundle_dir_if_necessary
|
||||||
|
build_agent_ctl
|
||||||
|
check_kata_runtime
|
||||||
|
if [ "${KUBERNETES}" == "yes" ]; then
|
||||||
|
init_kubernetes
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
rebuild_and_install_kata() {
|
||||||
|
check_out_repos
|
||||||
|
build_and_install_kata_runtime
|
||||||
|
build_and_add_agent_to_rootfs
|
||||||
|
build_and_install_rootfs
|
||||||
|
check_kata_runtime
|
||||||
|
}
|
||||||
|
|
||||||
|
# Based on the jenkins_job_build.sh script in kata-containers/tests/.ci - checks out source code and installs dependencies
|
||||||
|
initialize() {
|
||||||
|
# We need git to checkout and bootstrap the ci scripts
|
||||||
|
sudo apt-get update && sudo apt-get install -y git socat qemu-utils
|
||||||
|
|
||||||
|
PROFILE="${HOME}/.profile"
|
||||||
|
grep -qxF "export GOPATH=\${HOME}/go" "${PROFILE}" || echo "export GOPATH=\${HOME}/go" >> "${PROFILE}"
|
||||||
|
grep -qxF "export GOROOT=/usr/local/go" "${PROFILE}" || echo "export GOROOT=/usr/local/go" >> "${PROFILE}"
|
||||||
|
grep -qxF "export PATH=\${GOPATH}/bin:/usr/local/go/bin:/usr/sbin:/sbin:\${PATH}" "${PROFILE}" || echo "export PATH=\${GOPATH}/bin:/usr/local/go/bin:/usr/sbin:/sbin:\${PATH}" >> "${PROFILE}"
|
||||||
|
. "${HOME}/.profile"
|
||||||
|
mkdir -p "${GOPATH}"
|
||||||
|
|
||||||
|
check_out_repos
|
||||||
|
|
||||||
|
pushd "${tests_repo_dir}"
|
||||||
|
ci_dir_name=".ci"
|
||||||
|
"${ci_dir_name}/install_go.sh" -p -f
|
||||||
|
"${ci_dir_name}/install_rust.sh"
|
||||||
|
|
||||||
|
# Run setup, but don't install kata as we will build it ourselves in locations matching the developer guide
|
||||||
|
export INSTALL_KATA="no"
|
||||||
|
${ci_dir_name}/setup.sh
|
||||||
|
popd
|
||||||
|
}
|
||||||
|
|
||||||
|
check_out_repos() {
|
||||||
|
echo "Creating repo: ${tests_repo} and branch ${tests_branch} into ${tests_repo_dir}..."
|
||||||
|
mkdir -p $(dirname "${tests_repo_dir}") && sudo chown -R ${USER}:${USER} $(dirname "${tests_repo_dir}")
|
||||||
|
[ -d "${tests_repo_dir}" ] || git clone "https://${tests_repo}.git" "${tests_repo_dir}"
|
||||||
|
pushd "${tests_repo_dir}"
|
||||||
|
git fetch
|
||||||
|
if [ -n "${tests_branch}" ]; then
|
||||||
|
git checkout ${tests_branch}
|
||||||
|
fi
|
||||||
|
git reset --hard origin/${tests_branch}
|
||||||
|
popd
|
||||||
|
|
||||||
|
echo "Creating repo: ${katacontainers_repo} and branch ${katacontainers_branch} into ${katacontainers_repo_dir}..."
|
||||||
|
mkdir -p $(dirname "${katacontainers_repo_dir}") && sudo chown -R ${USER}:${USER} $(dirname "${katacontainers_repo_dir}")
|
||||||
|
[ -d "${katacontainers_repo_dir}" ] || git clone "https://${katacontainers_repo}.git" "${katacontainers_repo_dir}"
|
||||||
|
pushd "${katacontainers_repo_dir}"
|
||||||
|
git fetch
|
||||||
|
if [ -n "${katacontainers_branch}" ]; then
|
||||||
|
git checkout ${katacontainers_branch}
|
||||||
|
fi
|
||||||
|
git reset --hard origin/${katacontainers_branch}
|
||||||
|
popd
|
||||||
|
}
|
||||||
|
|
||||||
|
build_and_install_kata_runtime() {
|
||||||
|
cd ${katacontainers_repo_dir}/src/runtime
|
||||||
|
make clean && make && sudo -E PATH=$PATH make install
|
||||||
|
debug_output "We should have created Kata runtime binaries:: /usr/local/bin/kata-runtime and /usr/local/bin/containerd-shim-kata-v2"
|
||||||
|
debug_output "We should have made the Kata configuration file: /usr/share/defaults/kata-containers/configuration.toml"
|
||||||
|
debug_output "kata-runtime version: $(kata-runtime version)"
|
||||||
|
}
|
||||||
|
|
||||||
|
configure() {
|
||||||
|
debug_function configure_kata_to_use_rootfs
|
||||||
|
debug_function enable_full_debug
|
||||||
|
sudo systemctl restart containerd # Ensure containerd picks up debug configuration
|
||||||
|
}
|
||||||
|
|
||||||
|
configure_kata_to_use_rootfs() {
|
||||||
|
sudo mkdir -p /etc/kata-containers/
|
||||||
|
sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
|
||||||
|
sudo sed -i 's/^\(initrd =.*\)/# \1/g' /etc/kata-containers/configuration.toml
|
||||||
|
}
|
||||||
|
|
||||||
|
enable_full_debug() {
|
||||||
|
sudo mkdir -p /etc/kata-containers/
|
||||||
|
sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
|
||||||
|
|
||||||
|
# Note: if all enable_debug are set to true the agent console doesn't seem to work, so only enable the agent and runtime versions
|
||||||
|
# TODO LATER - try and work out why this is so we can replace the 2 lines below and stop it being so brittle sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = true/g' /etc/kata-containers/configuration.toml
|
||||||
|
sudo sed -z -i 's/\(# If enabled, make the agent display debug-level messages.\)\n\(# (default: disabled)\)\n#\(enable_debug = true\)\n/\1\n\2\n\3\n/' /etc/kata-containers/configuration.toml
|
||||||
|
sudo sed -z -i 's/\(# system log\)\n\(# (default: disabled)\)\n#\(enable_debug = true\)\n/\1\n\2\n\3\n/' /etc/kata-containers/configuration.toml
|
||||||
|
|
||||||
|
sudo sed -i -e 's/^# *\(debug_console_enabled\).*=.*$/\1 = true/g' /etc/kata-containers/configuration.toml
|
||||||
|
sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.log=debug initcall_debug"/g' /etc/kata-containers/configuration.toml
|
||||||
|
}
|
||||||
|
|
||||||
|
build_and_add_agent_to_rootfs() {
|
||||||
|
debug_function build_a_custom_kata_agent
|
||||||
|
debug_function add_custom_agent_to_rootfs
|
||||||
|
}
|
||||||
|
|
||||||
|
build_a_custom_kata_agent() {
|
||||||
|
if [[ ! -L /bin/musl-g++ ]]
|
||||||
|
then
|
||||||
|
rustup target add x86_64-unknown-linux-musl
|
||||||
|
sudo ln -s /usr/bin/g++ /bin/musl-g++
|
||||||
|
fi
|
||||||
|
. "$HOME/.cargo/env"
|
||||||
|
cd ${katacontainers_repo_dir}/src/agent && make
|
||||||
|
debug_output "Kata agent built: $(ls -al ${katacontainers_repo_dir}/src/agent/target/x86_64-unknown-linux-musl/release/kata-agent)"
|
||||||
|
# Run a make install into the rootfs directory in order to create the kata-agent.service file which is required when we add to the rootfs
|
||||||
|
sudo -E PATH=$PATH make install DESTDIR="${ROOTFS}"
|
||||||
|
}
|
||||||
|
|
||||||
|
create_a_local_rootfs() {
|
||||||
|
sudo rm -rf "${ROOTFS_DIR}"
|
||||||
|
cd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
|
||||||
|
runc_output=$(sudo docker info 2>/dev/null | grep -i "default runtime" | cut -d: -f2- | grep -q runc && echo "SUCCESS" || echo "ERROR: Incorrect default Docker runtime")
|
||||||
|
echo "Checking that runc is the default docker runtime: ${runc_output}"
|
||||||
|
export distro=fedora # I picked fedora as it supports s390x and x86_64 and uses the fedora registry, so we don't get DockerHub toomanyrequests issues
|
||||||
|
script -fec 'sudo -E GOPATH=$GOPATH EXTRA_PKGS="vim iputils net-tools iproute skopeo gnupg gpgme-devel" USE_DOCKER=true SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro}'
|
||||||
|
|
||||||
|
# Add umoci binary - TODO LATER replace with rpm when available in fedora
|
||||||
|
arch=amd64
|
||||||
|
mkdir -p ${ROOTFS_DIR}/usr/local/bin/
|
||||||
|
sudo curl -Lo ${ROOTFS_DIR}/usr/local/bin/umoci https://github.com/opencontainers/umoci/releases/download/v0.4.7/umoci.${arch}
|
||||||
|
sudo chmod u+x ${ROOTFS_DIR}/usr/local/bin/umoci
|
||||||
|
|
||||||
|
# During the ./rootfs.sh call the kata agent is built as root, so we need to update the permissions, so we can rebuild it
|
||||||
|
sudo chown -R ${USER}:${USER} "${katacontainers_repo_dir}/src/agent/"
|
||||||
|
}
|
||||||
|
|
||||||
|
add_custom_agent_to_rootfs() {
|
||||||
|
cd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
|
||||||
|
sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent
|
||||||
|
sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
|
||||||
|
sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
|
||||||
|
debug_output "Added kata agent to rootfs: $(ls -al ${ROOTFS_DIR}/usr/bin/kata-agent)"
|
||||||
|
}
|
||||||
|
|
||||||
|
build_and_install_rootfs() {
|
||||||
|
debug_function build_rootfs_image
|
||||||
|
debug_function install_rootfs_image
|
||||||
|
}
|
||||||
|
|
||||||
|
build_rootfs_image() {
|
||||||
|
cd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
|
||||||
|
script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}'
|
||||||
|
}
|
||||||
|
|
||||||
|
install_rootfs_image() {
|
||||||
|
cd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
|
||||||
|
commit=$(git log --format=%h -1 HEAD)
|
||||||
|
date=$(date +%Y-%m-%d-%T.%N%z)
|
||||||
|
image="kata-containers-${date}-${commit}"
|
||||||
|
sudo install -o root -g root -m 0640 -D kata-containers.img "/usr/share/kata-containers/${image}"
|
||||||
|
(cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
|
||||||
|
echo "Built Rootfs from ${ROOTFS_DIR} to /usr/share/kata-containers/${image}"
|
||||||
|
ls -al /usr/share/kata-containers/
|
||||||
|
}
|
||||||
|
|
||||||
|
install_guest_kernel_image() {
|
||||||
|
cd ${katacontainers_repo_dir}/tools/packaging/kernel
|
||||||
|
./build-kernel.sh setup
|
||||||
|
./build-kernel.sh build
|
||||||
|
sudo chmod 777 /usr/share/kata-containers/ # Give user permission to install kernel
|
||||||
|
./build-kernel.sh install
|
||||||
|
debug_output "New kernel installed to $(ls -al /usr/share/kata-containers/vmlinux*)"
|
||||||
|
}
|
||||||
|
|
||||||
|
build_qemu() {
|
||||||
|
${tests_repo_dir}/.ci/install_qemu.sh
|
||||||
|
}
|
||||||
|
|
||||||
|
check_kata_runtime() {
|
||||||
|
sudo kata-runtime check
|
||||||
|
}
|
||||||
|
|
||||||
|
init_kubernetes() {
|
||||||
|
export CI="true" && ${tests_repo_dir}/integration/kubernetes/init.sh
|
||||||
|
cat << EOT | tee ~/nginx-kata.yaml
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: nginx-kata
|
||||||
|
spec:
|
||||||
|
runtimeClassName: kata
|
||||||
|
containers:
|
||||||
|
- name: nginx
|
||||||
|
image: nginx
|
||||||
|
EOT
|
||||||
|
}
|
||||||
|
|
||||||
|
create_kata_pod() {
|
||||||
|
kubectl apply -f ~/nginx-kata.yaml
|
||||||
|
kubectl get pods
|
||||||
|
}
|
||||||
|
|
||||||
|
delete_kata_pod() {
|
||||||
|
kubectl delete -f ~/nginx-kata.yaml
|
||||||
|
}
|
||||||
|
|
||||||
|
test_kata_runtime() {
|
||||||
|
echo "Running ctr with the kata runtime..."
|
||||||
|
test_image="docker.io/library/busybox:latest"
|
||||||
|
sudo ctr image pull "${test_image}"
|
||||||
|
# If you hit too many requests run `sudo ctr image pull "docker.io/library/busybox:latest" -u <dockerhub username>` command and retry
|
||||||
|
sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "${test_image}" test-kata uname -a
|
||||||
|
}
|
||||||
|
|
||||||
|
run_kata_and_capture_logs() {
|
||||||
|
echo "Clearing systemd journal..."
|
||||||
|
sudo systemctl stop systemd-journald
|
||||||
|
sudo rm -f /var/log/journal/*/* /run/log/journal/*/*
|
||||||
|
sudo systemctl start systemd-journald
|
||||||
|
test_kata_runtime
|
||||||
|
echo "Collecting logs..."
|
||||||
|
sudo journalctl -q -o cat -a -t kata-runtime > ~/kata-runtime.log
|
||||||
|
sudo journalctl -q -o cat -a -t kata > ~/shimv2.log
|
||||||
|
echo "Logs output to ~/kata-runtime.log and ~/shimv2.log"
|
||||||
|
}
|
||||||
|
|
||||||
|
get_ids() {
|
||||||
|
guest_cid=$(ps -ef | grep qemu-system-x86_64 | egrep -o "guest-cid=[0-9]*" | cut -d= -f2) && sandbox_id=$(ps -ef | grep qemu | egrep -o "sandbox-[^,][^,]*" | sed 's/sandbox-//g' | awk '{print $1}')
|
||||||
|
}
|
||||||
|
|
||||||
|
open_kata_console() {
|
||||||
|
get_ids
|
||||||
|
sudo -E sandbox_id=${sandbox_id} su -c 'cd /var/run/vc/vm/${sandbox_id} && socat "stdin,raw,echo=0,escape=0x11" "unix-connect:console.sock"'
|
||||||
|
}
|
||||||
|
|
||||||
|
open_kata_shell() {
|
||||||
|
get_ids
|
||||||
|
sudo kata-runtime exec ${sandbox_id}
|
||||||
|
}
|
||||||
|
|
||||||
|
build_bundle_dir_if_necessary() {
|
||||||
|
bundle_dir="/tmp/bundle"
|
||||||
|
if [ ! -d "${bundle_dir}" ]; then
|
||||||
|
rootfs_dir="$bundle_dir/rootfs"
|
||||||
|
image="busybox"
|
||||||
|
mkdir -p "$rootfs_dir" && (cd "$bundle_dir" && runc spec)
|
||||||
|
sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf -
|
||||||
|
fi
|
||||||
|
# TODO - modify cat /tmp/bundle/config.json
|
||||||
|
# "args": [
|
||||||
|
# "sh" -> "/bin/sh"
|
||||||
|
}
|
||||||
|
|
||||||
|
build_agent_ctl() {
|
||||||
|
cd ${GOPATH}/src/${katacontainers_repo}/tools/agent-ctl/
|
||||||
|
sudo chown -R ${USER}:${USER} "${HOME}/.cargo/registry"
|
||||||
|
make
|
||||||
|
}
|
||||||
|
|
||||||
|
agent_pull_image() {
|
||||||
|
get_ids
|
||||||
|
build_bundle_dir_if_necessary
|
||||||
|
build_agent_ctl
|
||||||
|
./target/x86_64-unknown-linux-musl/release/kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c Check -c GetGuestDetails -c 'PullImage image=registry.fedoraproject.org/fedora:latest cid=01234567889'
|
||||||
|
#./target/x86_64-unknown-linux-musl/release/kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c Check -c GetGuestDetails -c 'PullImage image=docker.io/library/busybox:latest src_creds=<>'
|
||||||
|
}
|
||||||
|
|
||||||
|
create_container_command() {
|
||||||
|
get_ids
|
||||||
|
build_bundle_dir_if_necessary
|
||||||
|
# If kata-agent-ctl pre-built in this directory, use it directly
|
||||||
|
if [ -x kata-agent-ctl ]; then
|
||||||
|
./kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c Check -c GetGuestDetails -c 'CreateContainer cid=0123456789'
|
||||||
|
else
|
||||||
|
build_agent_ctl
|
||||||
|
./target/debug/kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c Check -c GetGuestDetails -c 'CreateContainer cid=0123456789'
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
run_agent_ctl_command() {
|
||||||
|
command=$1
|
||||||
|
# If kata-agent-ctl pre-built in this directory, use it directly
|
||||||
|
if [ -x kata-agent-ctl ]; then
|
||||||
|
./kata-agent-ctl ${command}
|
||||||
|
else
|
||||||
|
build_agent_ctl
|
||||||
|
./target/x86_64-unknown-linux-musl/release/kata-agent-ctl ${command}
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
agent_list_commands() {
|
||||||
|
run_agent_ctl_command "-l debug connect --bundle-dir \"\" --server-address \"\" -c list"
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
while getopts "dh" opt; do
|
||||||
|
case "$opt" in
|
||||||
|
d)
|
||||||
|
DEBUG="-d"
|
||||||
|
;;
|
||||||
|
h)
|
||||||
|
usage 0
|
||||||
|
;;
|
||||||
|
\?)
|
||||||
|
echo "Invalid option: -$OPTARG" >&2
|
||||||
|
usage 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
shift $((OPTIND - 1))
|
||||||
|
|
||||||
|
subcmd="${1:-}"
|
||||||
|
|
||||||
|
[ -z "${subcmd}" ] && usage 1
|
||||||
|
|
||||||
|
case "${subcmd}" in
|
||||||
|
all)
|
||||||
|
build_and_install_all
|
||||||
|
run_kata_and_capture_logs
|
||||||
|
;;
|
||||||
|
build_and_install_all)
|
||||||
|
build_and_install_all
|
||||||
|
;;
|
||||||
|
rebuild_and_install_kata)
|
||||||
|
rebuild_and_install_kata
|
||||||
|
;;
|
||||||
|
initialize)
|
||||||
|
initialize
|
||||||
|
;;
|
||||||
|
build_kata_runtime)
|
||||||
|
build_and_install_kata_runtime
|
||||||
|
;;
|
||||||
|
configure)
|
||||||
|
configure
|
||||||
|
;;
|
||||||
|
create_rootfs)
|
||||||
|
create_a_local_rootfs
|
||||||
|
;;
|
||||||
|
build_and_add_agent_to_rootfs)
|
||||||
|
build_and_add_agent_to_rootfs
|
||||||
|
;;
|
||||||
|
build_and_install_rootfs)
|
||||||
|
build_and_install_rootfs
|
||||||
|
;;
|
||||||
|
install_guest_kernel)
|
||||||
|
install_guest_kernel_image
|
||||||
|
;;
|
||||||
|
build_qemu)
|
||||||
|
build_qemu
|
||||||
|
;;
|
||||||
|
init_kubernetes)
|
||||||
|
init_kubernetes
|
||||||
|
;;
|
||||||
|
create_kata_pod)
|
||||||
|
create_kata_pod
|
||||||
|
;;
|
||||||
|
delete_kata_pod)
|
||||||
|
delete_kata_pod
|
||||||
|
;;
|
||||||
|
test)
|
||||||
|
test_kata_runtime
|
||||||
|
;;
|
||||||
|
test_capture_logs)
|
||||||
|
run_kata_and_capture_logs
|
||||||
|
;;
|
||||||
|
open_kata_console)
|
||||||
|
open_kata_console
|
||||||
|
;;
|
||||||
|
open_kata_shell)
|
||||||
|
open_kata_shell
|
||||||
|
;;
|
||||||
|
agent_pull_image)
|
||||||
|
agent_pull_image
|
||||||
|
;;
|
||||||
|
agent_list_commands)
|
||||||
|
agent_list_commands
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
usage 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
main $@
|
||||||
157
docs/how-to/how-to-build-and-test-ccv0.md
Normal file
157
docs/how-to/how-to-build-and-test-ccv0.md
Normal file
@ -0,0 +1,157 @@
|
|||||||
|
# How to build, run and test Kata CCv0
|
||||||
|
|
||||||
|
## Introduction and Background
|
||||||
|
|
||||||
|
In order to try and make building (locally) and demoing the Kata Containers `CCv0` code base as simple as possible I've shared a script [`ccv0.sh`](./ccv0.sh). This script was originally my attempt to automate the steps of the [Developer Guide](https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md) so that I could do different sections of them repeatedly and reliably as I was playing around with make changes to different parts of the Kata code base. I then tried to weave in some of the [`tests/.ci`](https://github.com/kata-containers/tests/tree/main/.ci) scripts in order to have less duplicated code and to make it support for platforms. Finally I extended it to include some calls to start kata pods in Kubernetes and call [`agent-ctl`](https://github.com/kata-containers/kata-containers/tree/main/tools/agent-ctl) to test the agent endpoint for pull image on guest for the CCv0 roadmap.
|
||||||
|
|
||||||
|
At the time of writing we only have some basic Kata agent pull image support for CCv0 included into the [`CCv0` branch](https://github.com/kata-containers/kata-containers/tree/CCv0), so the testing is limited to this, but as more functionality is added I'm hoping that this script can grow and expand to handle it.
|
||||||
|
|
||||||
|
*Disclaimer: This script has mostly just been used and tested by me ([@stevenhorsman](https://github.com/stevenhorsman)), so there might be issues with it. I'm happy to try and help solve these if possible, but this shouldn't be considered a fully supported process by the Kata Containers community.*
|
||||||
|
|
||||||
|
## Basic demo How-to
|
||||||
|
|
||||||
|
In order to build, and demo the CCv0 functionality, these are the steps I take:
|
||||||
|
- Provision a new VM
|
||||||
|
- *I choose a Ubuntu 20.04 8GB VM for this as I had one available. I think that the only part of the script that is OS dependent is the install of `git`, `socat` and `qemu-utils`(optional) using apt-get to bootstrap the rest of the installs. In order to run this on any platform just use your package manager to install these before running the `ccv0.sh` script and comment out the apt-get line with `sudo sed -i -e 's/\(sudo apt-get update .*\)$/# \1/g' ccv0.sh`*.
|
||||||
|
- Copy the script over to your VM *(I put it in the home directory)* and ensure it has execute permission by running `chmod u+x ccv0.sh`
|
||||||
|
- Optionally set up some environment variables
|
||||||
|
- By default the script checks out the `CCv0` branches of the `kata-containers/kata-containers` and `kata-containers/tests` repositories, but it is designed to be used to test of personal forks and branches as well. If you want to build and run these you can export the `katacontainers_repo`, `katacontainers_branch`, `tests_repo` and `tests_branch` variables e.g. `export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=stevenh/agent-pull-image-endpoint && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=stevenh/add-ccvo-changes-to-build` before running the script.
|
||||||
|
- Run the full build process with `. ~/ccv0.sh -d build_and_install_all`
|
||||||
|
- *I run this script sourced just so that the required installed components are accessible on the `PATH` to the rest of the process without having to reload the session.*
|
||||||
|
- The steps that `build_and_install_all` takes is:
|
||||||
|
- Checkout the git repos for the `tests` and `kata-containers` repos as specified by the environment variables (default to `CCv0` branches if they are not supplied)
|
||||||
|
- Use the `tests/.ci` scripts to install the build dependencies
|
||||||
|
- Build and install the Kata runtime
|
||||||
|
- Configure Kata to use containerd and for debug to be enabled (including enabling console access to the kata-runtime, which should only be done in development)
|
||||||
|
- Create, build and install a rootfs for the Kata hypervisor to use. For 'CCv0' this is currently based on Fedora 34 and has extra packages like `skopeo` and `umoci` added.
|
||||||
|
- Build the Kata guest kernel
|
||||||
|
- Install QEMU
|
||||||
|
- Set up `agent-ctl` testing by building the binary and configuring a bundle directory for it
|
||||||
|
- Initialising Kubernetes to use the VM as a single node cluster
|
||||||
|
- The first time this runs it may take a while, but subsequent runs will be quicker as more things are already installed and they can be further cut down by not running all the above steps [see "Additional script usage" below](#additional-script-usage)
|
||||||
|
- *Depending on how where your VMs are and how IPs are shared you might possibly get an error during "Store custom stress image in registry" from docker matching `ERROR: toomanyrequests: Too Many Requests`. In order to get around this log into docker hub with `sudo docker login` and re-run the step with `. ~/ccv0.sh -d init_kubernetes`.*
|
||||||
|
- Check that your Kubernetes cluster has been correctly set-up:
|
||||||
|
```
|
||||||
|
$ kubectl get nodes
|
||||||
|
NAME STATUS ROLES AGE VERSION
|
||||||
|
stevenh-ccv0-demo1.fyre.ibm.com Ready control-plane,master 3m33s v1.21.1
|
||||||
|
```
|
||||||
|
- Create a kata pod:
|
||||||
|
```
|
||||||
|
$ ~/ccv0.sh -d create_kata_pod
|
||||||
|
pod/nginx-kata created
|
||||||
|
NAME READY STATUS RESTARTS AGE
|
||||||
|
nginx-kata 0/1 ContainerCreating 0 5s
|
||||||
|
```
|
||||||
|
- Wait a few seconds for pod to start
|
||||||
|
```
|
||||||
|
$ kubectl get pods
|
||||||
|
NAME READY STATUS RESTARTS AGE
|
||||||
|
nginx-kata 1/1 Running 0 29s
|
||||||
|
```
|
||||||
|
- Create a new terminal to the VM and open shell into kata container and check the `/run/kata-containers` directory doesn't have a bundle unpack for container id `0123456789`:
|
||||||
|
```
|
||||||
|
$ ~/ccv0.sh -d open_kata_shell
|
||||||
|
bash-5.1# ls -al /run/kata-containers/
|
||||||
|
total 0
|
||||||
|
drwxr-xr-x 6 root root 120 Sep 6 09:44 .
|
||||||
|
drwxr-xr-x 8 root root 180 Sep 6 09:44 ..
|
||||||
|
drwxr-xr-x 3 root root 100 Sep 6 09:44 970af18fcef7e6e6f89fe1c4e77c23d647e18fae93b66303217e5d15996282d9
|
||||||
|
drwxr-xr-x 3 root root 100 Sep 6 09:44 ad20b902eb7fdf7b33dd6ca47e6c7805e2dcfcd534530f68a1b9e4973572ce1a
|
||||||
|
drwxr-xr-x 3 root root 80 Sep 6 09:44 sandbox
|
||||||
|
drwxr-xr-x 3 root root 60 Sep 6 09:44 shared
|
||||||
|
```
|
||||||
|
- In another new terminal open the kata console log for streaming:
|
||||||
|
```
|
||||||
|
$ ~/ccv0.sh -d open_kata_console
|
||||||
|
```
|
||||||
|
- In the first console list run the pull image agent endpoint using `~/ccv0.sh -d agent_pull_image`:
|
||||||
|
- *For unknown reasons sometimes the unpack fails the first time and the sandbox crashes, but seems to work the second time and the pod will restart automatically, so just re-open the shell and console and re-run the agent_pull_image.*
|
||||||
|
```
|
||||||
|
$ ~/ccv0.sh -d agent_pull_image
|
||||||
|
Finished release [optimized] target(s) in 0.21s
|
||||||
|
{"msg":"announce","level":"INFO","ts":"2021-09-06T02:48:26.612401254-07:00","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","pid":"129599","version":"0.1.0","config":"Config { server_address: \"vsock://1321076924:1024\", bundle_dir: \"/tmp/bundle\", timeout_nano: 0, interactive: false, ignore_errors: false }"}
|
||||||
|
{"msg":"client setup complete","level":"INFO","ts":"2021-09-06T02:48:26.618215437-07:00","source":"kata-agent-ctl","subsystem":"rpc","pid":"129599","name":"kata-agent-ctl","version":"0.1.0","server-address":"vsock://1321076924:1024"}
|
||||||
|
{"msg":"Run command Check (1 of 1)","level":"INFO","ts":"2021-09-06T02:48:26.618286397-07:00","name":"kata-agent-ctl","version":"0.1.0","subsystem":"rpc","source":"kata-agent-ctl","pid":"129599"}
|
||||||
|
{"msg":"response received","level":"INFO","ts":"2021-09-06T02:48:26.619212840-07:00","version":"0.1.0","subsystem":"rpc","pid":"129599","source":"kata-agent-ctl","name":"kata-agent-ctl","response":"status: SERVING"}
|
||||||
|
{"msg":"Command Check (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-06T02:48:26.619281890-07:00","subsystem":"rpc","name":"kata-agent-ctl","pid":"129599","source":"kata-agent-ctl","version":"0.1.0"}
|
||||||
|
{"msg":"Run command GetGuestDetails (1 of 1)","level":"INFO","ts":"2021-09-06T02:48:26.619328342-07:00","pid":"129599","version":"0.1.0","subsystem":"rpc","source":"kata-agent-ctl","name":"kata-agent-ctl"}
|
||||||
|
{"msg":"response received","level":"INFO","ts":"2021-09-06T02:48:26.622968404-07:00","pid":"129599","version":"0.1.0","subsystem":"rpc","name":"kata-agent-ctl","source":"kata-agent-ctl","response":"mem_block_size_bytes: 134217728 agent_details {version: \"2.3.0-alpha0\" storage_handlers: \"blk\" storage_handlers: \"9p\" storage_handlers: \"virtio-fs\" storage_handlers: \"ephemeral\" storage_handlers: \"mmioblk\" storage_handlers: \"local\" storage_handlers: \"scsi\" storage_handlers: \"nvdimm\" storage_handlers: \"watchable-bind\"}"}
|
||||||
|
{"msg":"Command GetGuestDetails (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-06T02:48:26.623049042-07:00","name":"kata-agent-ctl","pid":"129599","source":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0"}
|
||||||
|
{"msg":"Run command PullImage (1 of 1)","level":"INFO","ts":"2021-09-06T02:48:26.623081584-07:00","subsystem":"rpc","pid":"129599","name":"kata-agent-ctl","source":"kata-agent-ctl","version":"0.1.0"}
|
||||||
|
{"msg":"response received","level":"INFO","ts":"2021-09-06T02:48:54.270118679-07:00","subsystem":"rpc","version":"0.1.0","source":"kata-agent-ctl","name":"kata-agent-ctl","pid":"129599","response":""}
|
||||||
|
{"msg":"Command PullImage (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-06T02:48:54.270228983-07:00","pid":"129599","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0"}
|
||||||
|
```
|
||||||
|
- In the kata shell terminal you can see the container bundle has been created:
|
||||||
|
```
|
||||||
|
$ ls -al /run/kata-containers/01234567889
|
||||||
|
total 1216
|
||||||
|
drwx------ 3 root root 120 Sep 6 09:48 .
|
||||||
|
drwxr-xr-x 7 root root 140 Sep 6 09:48 ..
|
||||||
|
-rw-r--r-- 1 root root 3088 Sep 6 09:48 config.json
|
||||||
|
dr-xr-xr-x 18 root root 440 Aug 9 05:48 rootfs
|
||||||
|
-rw-r--r-- 1 root root 1235681 Sep 6 09:48 sha256_6db7cf62a51ac7d5b573f7a61a855093ff82d7c1caaf1413e7b4730a20a172d0.mtree
|
||||||
|
-rw-r--r-- 1 root root 372 Sep 6 09:48 umoci.json
|
||||||
|
```
|
||||||
|
- The console shell shows what has happened:
|
||||||
|
```
|
||||||
|
{"msg":"get guest details!","level":"INFO","ts":"2021-09-06T09:48:26.561831537+00:00","subsystem":"rpc","pid":"56","name":"kata-agent","source":"agent","version":"0.1.0"}
|
||||||
|
Getting image source signatures
|
||||||
|
…
|
||||||
|
…
|
||||||
|
Writing manifest to image destination
|
||||||
|
Storing signatures
|
||||||
|
…
|
||||||
|
…
|
||||||
|
Writing manifest to image destination
|
||||||
|
Storing signatures
|
||||||
|
• unpacking bundle ...
|
||||||
|
• unpack rootfs: /run/kata-containers/01234567889/rootfs
|
||||||
|
• unpack layer: sha256:ecfb9899f4ce3412a027b88f47dfea56664b5d4bc35eaa0f12c94c671f8ba503
|
||||||
|
• ... done
|
||||||
|
• computing filesystem manifest ...
|
||||||
|
• ... done
|
||||||
|
• unpacked image bundle: /run/kata-containers/01234567889
|
||||||
|
{"msg":"cid is \"01234567889\"","level":"INFO","ts":"2021-09-06T09:48:42.738268945+00:00","source":"agent","name":"kata-agent","subsystem":"rpc","pid":"56","version":"0.1.0"}
|
||||||
|
{"msg":"target_path_bundle is \"/run/kata-containers/01234567889\"","level":"INFO","ts":"2021-09-06T09:48:42.738355998+00:00","name":"kata-agent","source":"agent","pid":"56","subsystem":"rpc","version":"0.1.0"}
|
||||||
|
{"msg":"handling signal","level":"INFO","ts":"2021-09-06T09:48:54.212793601+00:00","name":"kata-agent","version":"0.1.0","pid":"56","subsystem":"signals","source":"agent","signal":"SIGCHLD"}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Additional script usage
|
||||||
|
|
||||||
|
As well as being able to use the script as above to build all of kata-containers from scratch it can be used to just re-build bits of it by running the script with different parameters. For example after the first build you will often not need to re-install the dependencies, QEMU or the Guest kernel, but just test code changes made to the runtime and agent. This can be done by running `. ~/ccv0.sh -d rebuild_and_install_kata` (*Note this re-does the checkout from git, do you changes are only made locally it is better to do the individual steps e.g. `. ~/ccv0.sh -d build_kata_runtime && . ~/ccv0.sh -d build_and_add_agent_to_rootfs && . ~/ccv0.sh -d build_and_install_rootfs`).* There are commands for a lot of steps in building, setting up and testing and the full list can be seen by running `~/ccv0.sh help`:
|
||||||
|
```
|
||||||
|
$ ~/ccv0.sh help
|
||||||
|
Overview:
|
||||||
|
Build and test kata containers from source
|
||||||
|
Optionally set kata-containers and tests repo and branch as exported variables before running
|
||||||
|
e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && . ~/ccv0.sh -d build_and_install_all
|
||||||
|
Usage:
|
||||||
|
ccv0.sh [options] <command>
|
||||||
|
Commands:
|
||||||
|
- help: Display this help
|
||||||
|
- all: Build and install everything, test kata with containerd and capture the logs
|
||||||
|
- build_and_install_all: Build and install everything
|
||||||
|
- initialise: Install dependencies and check out kata-containers source
|
||||||
|
- rebuild_and_install_kata: Rebuild the kata runtime and agent and build and install the image
|
||||||
|
- build_kata_runtime: Build and install the kata runtime
|
||||||
|
- configure: Configure Kata to use rootfs and enable debug
|
||||||
|
- create_rootfs: Create a local rootfs
|
||||||
|
- build_and_add_agent_to_rootfs:Builds the kata-agent and adds it to the rootfs
|
||||||
|
- build_and_install_rootfs: Builds and installs the rootfs image
|
||||||
|
- install_guest_kernel: Setup, build and install the guest kernel
|
||||||
|
- build_qemu: Checkout, patch, build and install QEMU
|
||||||
|
- init_kubernetes: initialise a Kubernetes cluster on this system
|
||||||
|
- create_kata_pod: Create a kata runtime nginx pod in Kubernetes
|
||||||
|
- delete_kata_pod: Delete a kata runtime nginx pod in Kubernetes
|
||||||
|
- open_kata_console: Stream the kata runtime's console
|
||||||
|
- open_kata_shell: Open a shell into the kata runtime
|
||||||
|
- agent_pull_image: Run PullImage command against the agent with agent-ctl
|
||||||
|
- agent_list_commands: List agent commands on agent-ctl
|
||||||
|
- test: Test using kata with containerd
|
||||||
|
- test_capture_logs: Test using kata with containerd and capture the logs in the user's home directory
|
||||||
|
|
||||||
|
Options:
|
||||||
|
-d: Enable debug
|
||||||
|
-h: Display this help
|
||||||
|
```
|
||||||
Loading…
Reference in New Issue
Block a user