diff --git a/docs/how-to/README.md b/docs/how-to/README.md index 44ffa21d3a..77246750f0 100644 --- a/docs/how-to/README.md +++ b/docs/how-to/README.md @@ -36,3 +36,6 @@ - [How to use hotplug memory on arm64 in Kata Containers](how-to-hotplug-memory-arm64.md) - [How to setup swap devices in guest kernel](how-to-setup-swap-devices-in-guest-kernel.md) - [How to run rootless vmm](how-to-run-rootless-vmm.md) + +## Confidential Containers +- [How to use build and test the Confidential Containers `CCv0` proof of concept](how-to-build-and-test-ccv0.md) diff --git a/docs/how-to/ccv0.sh b/docs/how-to/ccv0.sh new file mode 100755 index 0000000000..03b79ab8a6 --- /dev/null +++ b/docs/how-to/ccv0.sh @@ -0,0 +1,490 @@ +#!/bin/bash -e +# +# Copyright (c) 2021 IBM Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Disclaimer: This script is work in progress for supporting the CCv0 prototype +# It shouldn't be considered supported by the Kata Containers community, or anyone else + +# Based on https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md, but with elements of the tests/.ci scripts used + +readonly script_name="$(basename "${BASH_SOURCE[0]}")" + +# By default in Golang >= 1.16 GO111MODULE is set to "on", but not all modules support it, so overwrite to "auto" +export GO111MODULE="auto" + +# Setup kata containers environments if not set - we default to use containerd +export CRI_CONTAINERD=${CRI_CONTAINERD:-"yes"} +export CRI_RUNTIME=${CRI_RUNTIME:-"containerd"} +export CRIO=${CRIO:-"no"} +export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" +export KUBERNETES=${KUBERNETES:-"yes"} +export AGENT_INIT="${AGENT_INIT:-${TEST_INITRD:-no}}" + +# Allow the user to overwrite the default repo and branch names if they want to build from a fork +export katacontainers_repo="${katacontainers_repo:-github.com/kata-containers/kata-containers}" +export katacontainers_branch="${katacontainers_branch:-CCv0}" +export kata_default_branch=${katacontainers_branch} +export tests_repo="${tests_repo:-github.com/kata-containers/tests}" +export tests_branch="${tests_branch:-CCv0}" +export target_branch=${tests_branch} # kata-containers/ci/lib.sh uses target branch var to check out tests repo + +# Create a bunch of common, derived values up front so we don't need to create them in all the different functions +. "$HOME/.profile" +if [ -z ${GOPATH} ]; then + export GOPATH=${HOME}/go +fi +export tests_repo_dir="${GOPATH}/src/${tests_repo}" +export katacontainers_repo_dir="${GOPATH}/src/${katacontainers_repo}" +export ROOTFS_DIR="${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder/rootfs" + +debug_output() { + if [ -n "${DEBUG}" ] + then + echo "$(date): $@" + fi +} + +debug_function() { + debug_output "> $@" + start=$(date +%s%N | cut -b1-13) + $@; + status=$? + end=$(date +%s%N | cut -b1-13) + time=`expr ${end} - ${start}` + debug_output "< $@. Time taken: $(echo "scale=2; ${time} / 1000" | bc -l)s. RC: ${status}" +} + +usage() { + exit_code="$1" + cat < +Commands: +- help: Display this help +- all: Build and install everything, test kata with containerd and capture the logs +- build_and_install_all: Build and install everything +- initialize: Install dependencies and check out kata-containers source +- rebuild_and_install_kata: Rebuild the kata runtime and agent and build and install the image +- build_kata_runtime: Build and install the kata runtime +- configure: Configure Kata to use rootfs and enable debug +- create_rootfs: Create a local rootfs +- build_and_add_agent_to_rootfs:Builds the kata-agent and adds it to the rootfs +- build_and_install_rootfs: Builds and installs the rootfs image +- install_guest_kernel: Setup, build and install the guest kernel +- build_qemu: Checkout, patch, build and install QEMU +- init_kubernetes: initialize a Kubernetes cluster on this system +- create_kata_pod: Create a kata runtime nginx pod in Kubernetes +- delete_kata_pod: Delete a kata runtime nginx pod in Kubernetes +- open_kata_console: Stream the kata runtime's console +- open_kata_shell: Open a shell into the kata runtime +- agent_pull_image: Run PullImage command against the agent with agent-ctl +- agent_list_commands: List agent commands on agent-ctl +- test: Test using kata with containerd +- test_capture_logs: Test using kata with containerd and capture the logs in the user's home directory + +Options: + -d: Enable debug + -h: Display this help +EOT + # if script sourced don't exit as this will exit the main shell, just return instead + [[ $_ != $0 ]] && return "$exit_code" || exit "$exit_code" +} + +build_and_install_all() { + initialize + build_and_install_kata_runtime + configure + create_a_local_rootfs + build_and_install_rootfs + install_guest_kernel_image + build_qemu + build_bundle_dir_if_necessary + build_agent_ctl + check_kata_runtime + if [ "${KUBERNETES}" == "yes" ]; then + init_kubernetes + fi +} + +rebuild_and_install_kata() { + check_out_repos + build_and_install_kata_runtime + build_and_add_agent_to_rootfs + build_and_install_rootfs + check_kata_runtime +} + +# Based on the jenkins_job_build.sh script in kata-containers/tests/.ci - checks out source code and installs dependencies +initialize() { + # We need git to checkout and bootstrap the ci scripts + sudo apt-get update && sudo apt-get install -y git socat qemu-utils + + PROFILE="${HOME}/.profile" + grep -qxF "export GOPATH=\${HOME}/go" "${PROFILE}" || echo "export GOPATH=\${HOME}/go" >> "${PROFILE}" + grep -qxF "export GOROOT=/usr/local/go" "${PROFILE}" || echo "export GOROOT=/usr/local/go" >> "${PROFILE}" + grep -qxF "export PATH=\${GOPATH}/bin:/usr/local/go/bin:/usr/sbin:/sbin:\${PATH}" "${PROFILE}" || echo "export PATH=\${GOPATH}/bin:/usr/local/go/bin:/usr/sbin:/sbin:\${PATH}" >> "${PROFILE}" + . "${HOME}/.profile" + mkdir -p "${GOPATH}" + + check_out_repos + + pushd "${tests_repo_dir}" + ci_dir_name=".ci" + "${ci_dir_name}/install_go.sh" -p -f + "${ci_dir_name}/install_rust.sh" + + # Run setup, but don't install kata as we will build it ourselves in locations matching the developer guide + export INSTALL_KATA="no" + ${ci_dir_name}/setup.sh + popd +} + +check_out_repos() { + echo "Creating repo: ${tests_repo} and branch ${tests_branch} into ${tests_repo_dir}..." + mkdir -p $(dirname "${tests_repo_dir}") && sudo chown -R ${USER}:${USER} $(dirname "${tests_repo_dir}") + [ -d "${tests_repo_dir}" ] || git clone "https://${tests_repo}.git" "${tests_repo_dir}" + pushd "${tests_repo_dir}" + git fetch + if [ -n "${tests_branch}" ]; then + git checkout ${tests_branch} + fi + git reset --hard origin/${tests_branch} + popd + + echo "Creating repo: ${katacontainers_repo} and branch ${katacontainers_branch} into ${katacontainers_repo_dir}..." + mkdir -p $(dirname "${katacontainers_repo_dir}") && sudo chown -R ${USER}:${USER} $(dirname "${katacontainers_repo_dir}") + [ -d "${katacontainers_repo_dir}" ] || git clone "https://${katacontainers_repo}.git" "${katacontainers_repo_dir}" + pushd "${katacontainers_repo_dir}" + git fetch + if [ -n "${katacontainers_branch}" ]; then + git checkout ${katacontainers_branch} + fi + git reset --hard origin/${katacontainers_branch} + popd +} + +build_and_install_kata_runtime() { + cd ${katacontainers_repo_dir}/src/runtime + make clean && make && sudo -E PATH=$PATH make install + debug_output "We should have created Kata runtime binaries:: /usr/local/bin/kata-runtime and /usr/local/bin/containerd-shim-kata-v2" + debug_output "We should have made the Kata configuration file: /usr/share/defaults/kata-containers/configuration.toml" + debug_output "kata-runtime version: $(kata-runtime version)" +} + +configure() { + debug_function configure_kata_to_use_rootfs + debug_function enable_full_debug + sudo systemctl restart containerd # Ensure containerd picks up debug configuration +} + +configure_kata_to_use_rootfs() { + sudo mkdir -p /etc/kata-containers/ + sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers + sudo sed -i 's/^\(initrd =.*\)/# \1/g' /etc/kata-containers/configuration.toml +} + +enable_full_debug() { + sudo mkdir -p /etc/kata-containers/ + sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers + + # Note: if all enable_debug are set to true the agent console doesn't seem to work, so only enable the agent and runtime versions + # TODO LATER - try and work out why this is so we can replace the 2 lines below and stop it being so brittle sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = true/g' /etc/kata-containers/configuration.toml + sudo sed -z -i 's/\(# If enabled, make the agent display debug-level messages.\)\n\(# (default: disabled)\)\n#\(enable_debug = true\)\n/\1\n\2\n\3\n/' /etc/kata-containers/configuration.toml + sudo sed -z -i 's/\(# system log\)\n\(# (default: disabled)\)\n#\(enable_debug = true\)\n/\1\n\2\n\3\n/' /etc/kata-containers/configuration.toml + + sudo sed -i -e 's/^# *\(debug_console_enabled\).*=.*$/\1 = true/g' /etc/kata-containers/configuration.toml + sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.log=debug initcall_debug"/g' /etc/kata-containers/configuration.toml +} + +build_and_add_agent_to_rootfs() { + debug_function build_a_custom_kata_agent + debug_function add_custom_agent_to_rootfs +} + +build_a_custom_kata_agent() { + if [[ ! -L /bin/musl-g++ ]] + then + rustup target add x86_64-unknown-linux-musl + sudo ln -s /usr/bin/g++ /bin/musl-g++ + fi + . "$HOME/.cargo/env" + cd ${katacontainers_repo_dir}/src/agent && make + debug_output "Kata agent built: $(ls -al ${katacontainers_repo_dir}/src/agent/target/x86_64-unknown-linux-musl/release/kata-agent)" + # Run a make install into the rootfs directory in order to create the kata-agent.service file which is required when we add to the rootfs + sudo -E PATH=$PATH make install DESTDIR="${ROOTFS}" +} + +create_a_local_rootfs() { + sudo rm -rf "${ROOTFS_DIR}" + cd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder + runc_output=$(sudo docker info 2>/dev/null | grep -i "default runtime" | cut -d: -f2- | grep -q runc && echo "SUCCESS" || echo "ERROR: Incorrect default Docker runtime") + echo "Checking that runc is the default docker runtime: ${runc_output}" + export distro=fedora # I picked fedora as it supports s390x and x86_64 and uses the fedora registry, so we don't get DockerHub toomanyrequests issues + script -fec 'sudo -E GOPATH=$GOPATH EXTRA_PKGS="vim iputils net-tools iproute skopeo gnupg gpgme-devel" USE_DOCKER=true SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro}' + + # Add umoci binary - TODO LATER replace with rpm when available in fedora + arch=amd64 + mkdir -p ${ROOTFS_DIR}/usr/local/bin/ + sudo curl -Lo ${ROOTFS_DIR}/usr/local/bin/umoci https://github.com/opencontainers/umoci/releases/download/v0.4.7/umoci.${arch} + sudo chmod u+x ${ROOTFS_DIR}/usr/local/bin/umoci + + # During the ./rootfs.sh call the kata agent is built as root, so we need to update the permissions, so we can rebuild it + sudo chown -R ${USER}:${USER} "${katacontainers_repo_dir}/src/agent/" +} + +add_custom_agent_to_rootfs() { + cd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder + sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent + sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/ + sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/ + debug_output "Added kata agent to rootfs: $(ls -al ${ROOTFS_DIR}/usr/bin/kata-agent)" +} + +build_and_install_rootfs() { + debug_function build_rootfs_image + debug_function install_rootfs_image +} + +build_rootfs_image() { + cd ${katacontainers_repo_dir}/tools/osbuilder/image-builder + script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}' +} + +install_rootfs_image() { + cd ${katacontainers_repo_dir}/tools/osbuilder/image-builder + commit=$(git log --format=%h -1 HEAD) + date=$(date +%Y-%m-%d-%T.%N%z) + image="kata-containers-${date}-${commit}" + sudo install -o root -g root -m 0640 -D kata-containers.img "/usr/share/kata-containers/${image}" + (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img) + echo "Built Rootfs from ${ROOTFS_DIR} to /usr/share/kata-containers/${image}" + ls -al /usr/share/kata-containers/ +} + +install_guest_kernel_image() { + cd ${katacontainers_repo_dir}/tools/packaging/kernel + ./build-kernel.sh setup + ./build-kernel.sh build + sudo chmod 777 /usr/share/kata-containers/ # Give user permission to install kernel + ./build-kernel.sh install + debug_output "New kernel installed to $(ls -al /usr/share/kata-containers/vmlinux*)" +} + +build_qemu() { + ${tests_repo_dir}/.ci/install_qemu.sh +} + +check_kata_runtime() { + sudo kata-runtime check +} + +init_kubernetes() { + export CI="true" && ${tests_repo_dir}/integration/kubernetes/init.sh + cat << EOT | tee ~/nginx-kata.yaml +apiVersion: v1 +kind: Pod +metadata: + name: nginx-kata +spec: + runtimeClassName: kata + containers: + - name: nginx + image: nginx +EOT +} + +create_kata_pod() { + kubectl apply -f ~/nginx-kata.yaml + kubectl get pods +} + +delete_kata_pod() { + kubectl delete -f ~/nginx-kata.yaml +} + +test_kata_runtime() { + echo "Running ctr with the kata runtime..." + test_image="docker.io/library/busybox:latest" + sudo ctr image pull "${test_image}" + # If you hit too many requests run `sudo ctr image pull "docker.io/library/busybox:latest" -u ` command and retry + sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "${test_image}" test-kata uname -a +} + +run_kata_and_capture_logs() { + echo "Clearing systemd journal..." + sudo systemctl stop systemd-journald + sudo rm -f /var/log/journal/*/* /run/log/journal/*/* + sudo systemctl start systemd-journald + test_kata_runtime + echo "Collecting logs..." + sudo journalctl -q -o cat -a -t kata-runtime > ~/kata-runtime.log + sudo journalctl -q -o cat -a -t kata > ~/shimv2.log + echo "Logs output to ~/kata-runtime.log and ~/shimv2.log" +} + +get_ids() { + guest_cid=$(ps -ef | grep qemu-system-x86_64 | egrep -o "guest-cid=[0-9]*" | cut -d= -f2) && sandbox_id=$(ps -ef | grep qemu | egrep -o "sandbox-[^,][^,]*" | sed 's/sandbox-//g' | awk '{print $1}') +} + +open_kata_console() { + get_ids + sudo -E sandbox_id=${sandbox_id} su -c 'cd /var/run/vc/vm/${sandbox_id} && socat "stdin,raw,echo=0,escape=0x11" "unix-connect:console.sock"' +} + +open_kata_shell() { + get_ids + sudo kata-runtime exec ${sandbox_id} +} + +build_bundle_dir_if_necessary() { + bundle_dir="/tmp/bundle" + if [ ! -d "${bundle_dir}" ]; then + rootfs_dir="$bundle_dir/rootfs" + image="busybox" + mkdir -p "$rootfs_dir" && (cd "$bundle_dir" && runc spec) + sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf - + fi + # TODO - modify cat /tmp/bundle/config.json + # "args": [ + # "sh" -> "/bin/sh" +} + +build_agent_ctl() { + cd ${GOPATH}/src/${katacontainers_repo}/tools/agent-ctl/ + sudo chown -R ${USER}:${USER} "${HOME}/.cargo/registry" + make +} + +agent_pull_image() { + get_ids + build_bundle_dir_if_necessary + build_agent_ctl + ./target/x86_64-unknown-linux-musl/release/kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c Check -c GetGuestDetails -c 'PullImage image=registry.fedoraproject.org/fedora:latest cid=01234567889' + #./target/x86_64-unknown-linux-musl/release/kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c Check -c GetGuestDetails -c 'PullImage image=docker.io/library/busybox:latest src_creds=<>' +} + +create_container_command() { + get_ids + build_bundle_dir_if_necessary + # If kata-agent-ctl pre-built in this directory, use it directly + if [ -x kata-agent-ctl ]; then + ./kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c Check -c GetGuestDetails -c 'CreateContainer cid=0123456789' + else + build_agent_ctl + ./target/debug/kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c Check -c GetGuestDetails -c 'CreateContainer cid=0123456789' + fi +} + +run_agent_ctl_command() { + command=$1 + # If kata-agent-ctl pre-built in this directory, use it directly + if [ -x kata-agent-ctl ]; then + ./kata-agent-ctl ${command} + else + build_agent_ctl + ./target/x86_64-unknown-linux-musl/release/kata-agent-ctl ${command} + fi +} + +agent_list_commands() { + run_agent_ctl_command "-l debug connect --bundle-dir \"\" --server-address \"\" -c list" +} + +main() { + while getopts "dh" opt; do + case "$opt" in + d) + DEBUG="-d" + ;; + h) + usage 0 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + usage 1 + ;; + esac + done + + shift $((OPTIND - 1)) + + subcmd="${1:-}" + + [ -z "${subcmd}" ] && usage 1 + + case "${subcmd}" in + all) + build_and_install_all + run_kata_and_capture_logs + ;; + build_and_install_all) + build_and_install_all + ;; + rebuild_and_install_kata) + rebuild_and_install_kata + ;; + initialize) + initialize + ;; + build_kata_runtime) + build_and_install_kata_runtime + ;; + configure) + configure + ;; + create_rootfs) + create_a_local_rootfs + ;; + build_and_add_agent_to_rootfs) + build_and_add_agent_to_rootfs + ;; + build_and_install_rootfs) + build_and_install_rootfs + ;; + install_guest_kernel) + install_guest_kernel_image + ;; + build_qemu) + build_qemu + ;; + init_kubernetes) + init_kubernetes + ;; + create_kata_pod) + create_kata_pod + ;; + delete_kata_pod) + delete_kata_pod + ;; + test) + test_kata_runtime + ;; + test_capture_logs) + run_kata_and_capture_logs + ;; + open_kata_console) + open_kata_console + ;; + open_kata_shell) + open_kata_shell + ;; + agent_pull_image) + agent_pull_image + ;; + agent_list_commands) + agent_list_commands + ;; + *) + usage 1 + ;; + esac +} + +main $@ \ No newline at end of file diff --git a/docs/how-to/how-to-build-and-test-ccv0.md b/docs/how-to/how-to-build-and-test-ccv0.md new file mode 100644 index 0000000000..3d7a270576 --- /dev/null +++ b/docs/how-to/how-to-build-and-test-ccv0.md @@ -0,0 +1,157 @@ +# How to build, run and test Kata CCv0 + +## Introduction and Background + +In order to try and make building (locally) and demoing the Kata Containers `CCv0` code base as simple as possible I've shared a script [`ccv0.sh`](./ccv0.sh). This script was originally my attempt to automate the steps of the [Developer Guide](https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md) so that I could do different sections of them repeatedly and reliably as I was playing around with make changes to different parts of the Kata code base. I then tried to weave in some of the [`tests/.ci`](https://github.com/kata-containers/tests/tree/main/.ci) scripts in order to have less duplicated code and to make it support for platforms. Finally I extended it to include some calls to start kata pods in Kubernetes and call [`agent-ctl`](https://github.com/kata-containers/kata-containers/tree/main/tools/agent-ctl) to test the agent endpoint for pull image on guest for the CCv0 roadmap. + +At the time of writing we only have some basic Kata agent pull image support for CCv0 included into the [`CCv0` branch](https://github.com/kata-containers/kata-containers/tree/CCv0), so the testing is limited to this, but as more functionality is added I'm hoping that this script can grow and expand to handle it. + +*Disclaimer: This script has mostly just been used and tested by me ([@stevenhorsman](https://github.com/stevenhorsman)), so there might be issues with it. I'm happy to try and help solve these if possible, but this shouldn't be considered a fully supported process by the Kata Containers community.* + +## Basic demo How-to + +In order to build, and demo the CCv0 functionality, these are the steps I take: +- Provision a new VM + - *I choose a Ubuntu 20.04 8GB VM for this as I had one available. I think that the only part of the script that is OS dependent is the install of `git`, `socat` and `qemu-utils`(optional) using apt-get to bootstrap the rest of the installs. In order to run this on any platform just use your package manager to install these before running the `ccv0.sh` script and comment out the apt-get line with `sudo sed -i -e 's/\(sudo apt-get update .*\)$/# \1/g' ccv0.sh`*. +- Copy the script over to your VM *(I put it in the home directory)* and ensure it has execute permission by running `chmod u+x ccv0.sh` +- Optionally set up some environment variables + - By default the script checks out the `CCv0` branches of the `kata-containers/kata-containers` and `kata-containers/tests` repositories, but it is designed to be used to test of personal forks and branches as well. If you want to build and run these you can export the `katacontainers_repo`, `katacontainers_branch`, `tests_repo` and `tests_branch` variables e.g. `export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=stevenh/agent-pull-image-endpoint && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=stevenh/add-ccvo-changes-to-build` before running the script. +- Run the full build process with `. ~/ccv0.sh -d build_and_install_all` + - *I run this script sourced just so that the required installed components are accessible on the `PATH` to the rest of the process without having to reload the session.* + - The steps that `build_and_install_all` takes is: + - Checkout the git repos for the `tests` and `kata-containers` repos as specified by the environment variables (default to `CCv0` branches if they are not supplied) + - Use the `tests/.ci` scripts to install the build dependencies + - Build and install the Kata runtime + - Configure Kata to use containerd and for debug to be enabled (including enabling console access to the kata-runtime, which should only be done in development) + - Create, build and install a rootfs for the Kata hypervisor to use. For 'CCv0' this is currently based on Fedora 34 and has extra packages like `skopeo` and `umoci` added. + - Build the Kata guest kernel + - Install QEMU + - Set up `agent-ctl` testing by building the binary and configuring a bundle directory for it + - Initialising Kubernetes to use the VM as a single node cluster + - The first time this runs it may take a while, but subsequent runs will be quicker as more things are already installed and they can be further cut down by not running all the above steps [see "Additional script usage" below](#additional-script-usage) + - *Depending on how where your VMs are and how IPs are shared you might possibly get an error during "Store custom stress image in registry" from docker matching `ERROR: toomanyrequests: Too Many Requests`. In order to get around this log into docker hub with `sudo docker login` and re-run the step with `. ~/ccv0.sh -d init_kubernetes`.* +- Check that your Kubernetes cluster has been correctly set-up: +``` +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +stevenh-ccv0-demo1.fyre.ibm.com Ready control-plane,master 3m33s v1.21.1 +``` +- Create a kata pod: +``` +$ ~/ccv0.sh -d create_kata_pod +pod/nginx-kata created +NAME READY STATUS RESTARTS AGE +nginx-kata 0/1 ContainerCreating 0 5s +``` +- Wait a few seconds for pod to start +``` +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +nginx-kata 1/1 Running 0 29s +``` +- Create a new terminal to the VM and open shell into kata container and check the `/run/kata-containers` directory doesn't have a bundle unpack for container id `0123456789`: +``` +$ ~/ccv0.sh -d open_kata_shell +bash-5.1# ls -al /run/kata-containers/ +total 0 +drwxr-xr-x 6 root root 120 Sep 6 09:44 . +drwxr-xr-x 8 root root 180 Sep 6 09:44 .. +drwxr-xr-x 3 root root 100 Sep 6 09:44 970af18fcef7e6e6f89fe1c4e77c23d647e18fae93b66303217e5d15996282d9 +drwxr-xr-x 3 root root 100 Sep 6 09:44 ad20b902eb7fdf7b33dd6ca47e6c7805e2dcfcd534530f68a1b9e4973572ce1a +drwxr-xr-x 3 root root 80 Sep 6 09:44 sandbox +drwxr-xr-x 3 root root 60 Sep 6 09:44 shared +``` +- In another new terminal open the kata console log for streaming: +``` +$ ~/ccv0.sh -d open_kata_console +``` +- In the first console list run the pull image agent endpoint using `~/ccv0.sh -d agent_pull_image`: + - *For unknown reasons sometimes the unpack fails the first time and the sandbox crashes, but seems to work the second time and the pod will restart automatically, so just re-open the shell and console and re-run the agent_pull_image.* +``` +$ ~/ccv0.sh -d agent_pull_image + Finished release [optimized] target(s) in 0.21s +{"msg":"announce","level":"INFO","ts":"2021-09-06T02:48:26.612401254-07:00","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","pid":"129599","version":"0.1.0","config":"Config { server_address: \"vsock://1321076924:1024\", bundle_dir: \"/tmp/bundle\", timeout_nano: 0, interactive: false, ignore_errors: false }"} +{"msg":"client setup complete","level":"INFO","ts":"2021-09-06T02:48:26.618215437-07:00","source":"kata-agent-ctl","subsystem":"rpc","pid":"129599","name":"kata-agent-ctl","version":"0.1.0","server-address":"vsock://1321076924:1024"} +{"msg":"Run command Check (1 of 1)","level":"INFO","ts":"2021-09-06T02:48:26.618286397-07:00","name":"kata-agent-ctl","version":"0.1.0","subsystem":"rpc","source":"kata-agent-ctl","pid":"129599"} +{"msg":"response received","level":"INFO","ts":"2021-09-06T02:48:26.619212840-07:00","version":"0.1.0","subsystem":"rpc","pid":"129599","source":"kata-agent-ctl","name":"kata-agent-ctl","response":"status: SERVING"} +{"msg":"Command Check (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-06T02:48:26.619281890-07:00","subsystem":"rpc","name":"kata-agent-ctl","pid":"129599","source":"kata-agent-ctl","version":"0.1.0"} +{"msg":"Run command GetGuestDetails (1 of 1)","level":"INFO","ts":"2021-09-06T02:48:26.619328342-07:00","pid":"129599","version":"0.1.0","subsystem":"rpc","source":"kata-agent-ctl","name":"kata-agent-ctl"} +{"msg":"response received","level":"INFO","ts":"2021-09-06T02:48:26.622968404-07:00","pid":"129599","version":"0.1.0","subsystem":"rpc","name":"kata-agent-ctl","source":"kata-agent-ctl","response":"mem_block_size_bytes: 134217728 agent_details {version: \"2.3.0-alpha0\" storage_handlers: \"blk\" storage_handlers: \"9p\" storage_handlers: \"virtio-fs\" storage_handlers: \"ephemeral\" storage_handlers: \"mmioblk\" storage_handlers: \"local\" storage_handlers: \"scsi\" storage_handlers: \"nvdimm\" storage_handlers: \"watchable-bind\"}"} +{"msg":"Command GetGuestDetails (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-06T02:48:26.623049042-07:00","name":"kata-agent-ctl","pid":"129599","source":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0"} +{"msg":"Run command PullImage (1 of 1)","level":"INFO","ts":"2021-09-06T02:48:26.623081584-07:00","subsystem":"rpc","pid":"129599","name":"kata-agent-ctl","source":"kata-agent-ctl","version":"0.1.0"} +{"msg":"response received","level":"INFO","ts":"2021-09-06T02:48:54.270118679-07:00","subsystem":"rpc","version":"0.1.0","source":"kata-agent-ctl","name":"kata-agent-ctl","pid":"129599","response":""} +{"msg":"Command PullImage (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-06T02:48:54.270228983-07:00","pid":"129599","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0"} +``` +- In the kata shell terminal you can see the container bundle has been created: +``` +$ ls -al /run/kata-containers/01234567889 +total 1216 +drwx------ 3 root root 120 Sep 6 09:48 . +drwxr-xr-x 7 root root 140 Sep 6 09:48 .. +-rw-r--r-- 1 root root 3088 Sep 6 09:48 config.json +dr-xr-xr-x 18 root root 440 Aug 9 05:48 rootfs +-rw-r--r-- 1 root root 1235681 Sep 6 09:48 sha256_6db7cf62a51ac7d5b573f7a61a855093ff82d7c1caaf1413e7b4730a20a172d0.mtree +-rw-r--r-- 1 root root 372 Sep 6 09:48 umoci.json +``` +- The console shell shows what has happened: +``` +{"msg":"get guest details!","level":"INFO","ts":"2021-09-06T09:48:26.561831537+00:00","subsystem":"rpc","pid":"56","name":"kata-agent","source":"agent","version":"0.1.0"} +Getting image source signatures +… +… +Writing manifest to image destination +Storing signatures +… +… +Writing manifest to image destination +Storing signatures + • unpacking bundle ... + • unpack rootfs: /run/kata-containers/01234567889/rootfs + • unpack layer: sha256:ecfb9899f4ce3412a027b88f47dfea56664b5d4bc35eaa0f12c94c671f8ba503 + • ... done + • computing filesystem manifest ... + • ... done + • unpacked image bundle: /run/kata-containers/01234567889 +{"msg":"cid is \"01234567889\"","level":"INFO","ts":"2021-09-06T09:48:42.738268945+00:00","source":"agent","name":"kata-agent","subsystem":"rpc","pid":"56","version":"0.1.0"} +{"msg":"target_path_bundle is \"/run/kata-containers/01234567889\"","level":"INFO","ts":"2021-09-06T09:48:42.738355998+00:00","name":"kata-agent","source":"agent","pid":"56","subsystem":"rpc","version":"0.1.0"} +{"msg":"handling signal","level":"INFO","ts":"2021-09-06T09:48:54.212793601+00:00","name":"kata-agent","version":"0.1.0","pid":"56","subsystem":"signals","source":"agent","signal":"SIGCHLD"} +``` + +## Additional script usage + +As well as being able to use the script as above to build all of kata-containers from scratch it can be used to just re-build bits of it by running the script with different parameters. For example after the first build you will often not need to re-install the dependencies, QEMU or the Guest kernel, but just test code changes made to the runtime and agent. This can be done by running `. ~/ccv0.sh -d rebuild_and_install_kata` (*Note this re-does the checkout from git, do you changes are only made locally it is better to do the individual steps e.g. `. ~/ccv0.sh -d build_kata_runtime && . ~/ccv0.sh -d build_and_add_agent_to_rootfs && . ~/ccv0.sh -d build_and_install_rootfs`).* There are commands for a lot of steps in building, setting up and testing and the full list can be seen by running `~/ccv0.sh help`: +``` +$ ~/ccv0.sh help +Overview: + Build and test kata containers from source + Optionally set kata-containers and tests repo and branch as exported variables before running + e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && . ~/ccv0.sh -d build_and_install_all +Usage: + ccv0.sh [options] +Commands: +- help: Display this help +- all: Build and install everything, test kata with containerd and capture the logs +- build_and_install_all: Build and install everything +- initialise: Install dependencies and check out kata-containers source +- rebuild_and_install_kata: Rebuild the kata runtime and agent and build and install the image +- build_kata_runtime: Build and install the kata runtime +- configure: Configure Kata to use rootfs and enable debug +- create_rootfs: Create a local rootfs +- build_and_add_agent_to_rootfs:Builds the kata-agent and adds it to the rootfs +- build_and_install_rootfs: Builds and installs the rootfs image +- install_guest_kernel: Setup, build and install the guest kernel +- build_qemu: Checkout, patch, build and install QEMU +- init_kubernetes: initialise a Kubernetes cluster on this system +- create_kata_pod: Create a kata runtime nginx pod in Kubernetes +- delete_kata_pod: Delete a kata runtime nginx pod in Kubernetes +- open_kata_console: Stream the kata runtime's console +- open_kata_shell: Open a shell into the kata runtime +- agent_pull_image: Run PullImage command against the agent with agent-ctl +- agent_list_commands: List agent commands on agent-ctl +- test: Test using kata with containerd +- test_capture_logs: Test using kata with containerd and capture the logs in the user's home directory + +Options: + -d: Enable debug + -h: Display this help +``` \ No newline at end of file