github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-08-21 17:34:31 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #3564 from stevenhorsman/ccv0-improvements

Browse Source 
CCv0 | doc: Improve ccv0.sh and CCv0 how-to document
This commit is contained in:
Steve Horsman 2022-02-07 13:02:20 +00:00 committed by GitHub
commit e71592d5da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 167 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
docs/how-to/ccv0.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash -e #!/bin/bash -e
# #
# Copyright (c) 2021 IBM Corporation # Copyright (c) 2021, 2022 IBM Corporation
# #
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# #
@ -8,7 +8,8 @@
# Disclaimer: This script is work in progress for supporting the CCv0 prototype # Disclaimer: This script is work in progress for supporting the CCv0 prototype
# It shouldn't be considered supported by the Kata Containers community, or anyone else # It shouldn't be considered supported by the Kata Containers community, or anyone else
# Based on https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md, but with elements of the tests/.ci scripts used # Based on https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md,
# but with elements of the tests/.ci scripts used
readonly script_name="$(basename "${BASH_SOURCE[0]}")" readonly script_name="$(basename "${BASH_SOURCE[0]}")"
@ -31,8 +32,14 @@ export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
export tests_branch="${tests_branch:-CCv0}" export tests_branch="${tests_branch:-CCv0}"
export target_branch=${tests_branch} # kata-containers/ci/lib.sh uses target branch var to check out tests repo export target_branch=${tests_branch} # kata-containers/ci/lib.sh uses target branch var to check out tests repo
# if .bash_profile exists then use it, otherwise fall back to .profile
export PROFILE="${HOME}/.profile"
if [ -r "${HOME}/.bash_profile" ]; then
export PROFILE="${HOME}/.bash_profile"
fi
# Create a bunch of common, derived values up front so we don't need to create them in all the different functions # Create a bunch of common, derived values up front so we don't need to create them in all the different functions
. "$HOME/.profile" . ${PROFILE}
if [ -z ${GOPATH} ]; then if [ -z ${GOPATH} ]; then
export GOPATH=${HOME}/go export GOPATH=${HOME}/go
fi fi
@ -41,6 +48,8 @@ export katacontainers_repo_dir="${GOPATH}/src/${katacontainers_repo}"
export ROOTFS_DIR="${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder/rootfs" export ROOTFS_DIR="${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder/rootfs"
export PULL_IMAGE="${PULL_IMAGE:-quay.io/kata-containers/confidential-containers:signed}" # Doesn't need authentication export PULL_IMAGE="${PULL_IMAGE:-quay.io/kata-containers/confidential-containers:signed}" # Doesn't need authentication
export CONTAINER_ID="${CONTAINER_ID:-0123456789}" export CONTAINER_ID="${CONTAINER_ID:-0123456789}"
source /etc/os-release || source /usr/lib/os-release
grep -Eq "\<fedora\>" /etc/os-release 2> /dev/null && export USE_PODMAN=true
debug_output() { debug_output() {
if [ -n "${DEBUG}" ] if [ -n "${DEBUG}" ]
@ -65,7 +74,7 @@ usage() {
Overview: Overview:
Build and test kata containers from source Build and test kata containers from source
Optionally set kata-containers and tests repo and branch as exported variables before running Optionally set kata-containers and tests repo and branch as exported variables before running
e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && . ~/${script_name} -d build_and_install_all e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/${script_name} build_and_install_all
Usage: Usage:
${script_name} [options] <command> ${script_name} [options] <command>
Commands: Commands:
@ -129,26 +138,30 @@ rebuild_and_install_kata() {
# Based on the jenkins_job_build.sh script in kata-containers/tests/.ci - checks out source code and installs dependencies # Based on the jenkins_job_build.sh script in kata-containers/tests/.ci - checks out source code and installs dependencies
initialize() { initialize() {
# We need git to checkout and bootstrap the ci scripts # We need git to checkout and bootstrap the ci scripts and some other packages used in testing
sudo apt-get update && sudo apt-get install -y git socat qemu-utils sudo apt-get update && sudo apt-get install -y curl git socat qemu-utils
PROFILE="${HOME}/.profile" PROFILE="${HOME}/.profile"
grep -qxF "export GOPATH=\${HOME}/go" "${PROFILE}" || echo "export GOPATH=\${HOME}/go" >> "${PROFILE}" grep -qxF "export GOPATH=\${HOME}/go" "${PROFILE}" || echo "export GOPATH=\${HOME}/go" >> "${PROFILE}"
grep -qxF "export GOROOT=/usr/local/go" "${PROFILE}" || echo "export GOROOT=/usr/local/go" >> "${PROFILE}" grep -qxF "export GOROOT=/usr/local/go" "${PROFILE}" || echo "export GOROOT=/usr/local/go" >> "${PROFILE}"
grep -qxF "export PATH=\${GOPATH}/bin:/usr/local/go/bin:/usr/sbin:/sbin:\${PATH}" "${PROFILE}" || echo "export PATH=\${GOPATH}/bin:/usr/local/go/bin:/usr/sbin:/sbin:\${PATH}" >> "${PROFILE}" grep -qxF "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" "${PROFILE}" || echo "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" >> "${PROFILE}"
. "${HOME}/.profile"
# Load the new go and PATH parameters from the profile
. ${PROFILE}
mkdir -p "${GOPATH}" mkdir -p "${GOPATH}"
check_out_repos check_out_repos
pushd "${tests_repo_dir}" pushd "${tests_repo_dir}"
ci_dir_name=".ci" ci_dir_name=".ci"
"${ci_dir_name}/install_go.sh" -p -f sudo -E PATH=$PATH -s "${ci_dir_name}/install_go.sh" -p -f
"${ci_dir_name}/install_rust.sh" sudo -E PATH=$PATH -s "${ci_dir_name}/install_rust.sh"
# Run setup, but don't install kata as we will build it ourselves in locations matching the developer guide # Run setup, but don't install kata as we will build it ourselves in locations matching the developer guide
export INSTALL_KATA="no" export INSTALL_KATA="no"
${ci_dir_name}/setup.sh sudo -E PATH=$PATH -s ${ci_dir_name}/setup.sh
# Reload the profile to pick up installed dependencies
. ${PROFILE}
popd popd
} }
@ -177,11 +190,12 @@ check_out_repos() {
} }
build_and_install_kata_runtime() { build_and_install_kata_runtime() {
cd ${katacontainers_repo_dir}/src/runtime pushd ${katacontainers_repo_dir}/src/runtime
make clean && make && sudo -E PATH=$PATH make install make clean && make && sudo -E PATH=$PATH make install
debug_output "We should have created Kata runtime binaries:: /usr/local/bin/kata-runtime and /usr/local/bin/containerd-shim-kata-v2" debug_output "We should have created Kata runtime binaries:: /usr/local/bin/kata-runtime and /usr/local/bin/containerd-shim-kata-v2"
debug_output "We should have made the Kata configuration file: /usr/share/defaults/kata-containers/configuration.toml" debug_output "We should have made the Kata configuration file: /usr/share/defaults/kata-containers/configuration.toml"
debug_output "kata-runtime version: $(kata-runtime version)" debug_output "kata-runtime version: $(kata-runtime version)"
popd
} }
configure() { configure() {
@ -195,11 +209,11 @@ configure() {
# insert the cri_handler = "cc" into the [plugins.cri.containerd.runtimes.kata] section # insert the cri_handler = "cc" into the [plugins.cri.containerd.runtimes.kata] section
sudo sed -z -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' /etc/containerd/config.toml sudo sed -z -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' /etc/containerd/config.toml
fi fi
# Add cni directory to containerd config # Add cni directory to containerd config
echo " [plugins.cri.cni] echo " [plugins.cri.cni]
# conf_dir is the directory in which the admin places a CNI conf. # conf_dir is the directory in which the admin places a CNI conf.
conf_dir = \"/etc/cni/net.d\"" >> /etc/containerd/config.toml conf_dir = \"/etc/cni/net.d\"" | sudo tee -a /etc/containerd/config.toml
# Switch image offload to true in kata config # Switch image offload to true in kata config
sudo sed -i -e 's/^# *\(service_offload\).*=.*$/\1 = true/g' /etc/kata-containers/configuration.toml sudo sed -i -e 's/^# *\(service_offload\).*=.*$/\1 = true/g' /etc/kata-containers/configuration.toml
@ -232,40 +246,48 @@ build_and_add_agent_to_rootfs() {
} }
build_a_custom_kata_agent() { build_a_custom_kata_agent() {
if [[ ! -L /bin/musl-g++ ]]
then
rustup target add x86_64-unknown-linux-musl
sudo ln -s /usr/bin/g++ /bin/musl-g++
fi
. "$HOME/.cargo/env" . "$HOME/.cargo/env"
cd ${katacontainers_repo_dir}/src/agent && make pushd ${katacontainers_repo_dir}/src/agent
debug_output "Kata agent built: $(ls -al ${katacontainers_repo_dir}/src/agent/target/x86_64-unknown-linux-musl/release/kata-agent)" sudo -E PATH=$PATH make
ARCH=$(uname -m)
[ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
[ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
debug_output "Kata agent built: $(ls -al ${katacontainers_repo_dir}/src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent)"
# Run a make install into the rootfs directory in order to create the kata-agent.service file which is required when we add to the rootfs # Run a make install into the rootfs directory in order to create the kata-agent.service file which is required when we add to the rootfs
sudo -E PATH=$PATH make install DESTDIR="${ROOTFS}" sudo -E PATH=$PATH make install DESTDIR="${ROOTFS}"
popd
} }
create_a_local_rootfs() { create_a_local_rootfs() {
sudo rm -rf "${ROOTFS_DIR}" sudo rm -rf "${ROOTFS_DIR}"
cd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
export distro="ubuntu" export distro="ubuntu"
[[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}" [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
sudo -E OS_VERSION="${OS_VERSION:-}" GOPATH=$GOPATH DEBUG="${DEBUG}" USE_DOCKER="${use_docker:-}" SKOPEO=${SKOPEO:-} UMOCI=yes SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro} sudo -E OS_VERSION="${OS_VERSION:-}" GOPATH=$GOPATH EXTRA_PKGS="vim iputils-ping net-tools" DEBUG="${DEBUG}" USE_DOCKER="${use_docker:-}" SKOPEO=${SKOPEO:-} UMOCI=yes SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro}
# Install_rust.sh during rootfs.sh switches us to the main branch of the tests repo, so switch back now # Install_rust.sh during rootfs.sh switches us to the main branch of the tests repo, so switch back now
pushd "${tests_repo_dir}" pushd "${tests_repo_dir}"
git checkout ${tests_branch} git checkout ${tests_branch}
popd popd
# During the ./rootfs.sh call the kata agent is built as root, so we need to update the permissions, so we can rebuild it # During the ./rootfs.sh call the kata agent is built as root, so we need to update the permissions, so we can rebuild it
sudo chown -R ${USER}:${USER} "${katacontainers_repo_dir}/src/agent/" sudo chown -R ${USER}:${USER} "${katacontainers_repo_dir}/src/agent/"
popd
} }
add_custom_agent_to_rootfs() { add_custom_agent_to_rootfs() {
cd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent
ARCH=$(uname -m)
[ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
[ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ${katacontainers_repo_dir}/src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent
sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/ sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/ sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
debug_output "Added kata agent to rootfs: $(ls -al ${ROOTFS_DIR}/usr/bin/kata-agent)" debug_output "Added kata agent to rootfs: $(ls -al ${ROOTFS_DIR}/usr/bin/kata-agent)"
popd
} }
build_and_install_rootfs() { build_and_install_rootfs() {
@ -274,12 +296,15 @@ build_and_install_rootfs() {
} }
build_rootfs_image() { build_rootfs_image() {
cd ${katacontainers_repo_dir}/tools/osbuilder/image-builder pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}' # Logic from install_kata_image.sh - if we aren't using podman (ie on a fedora like), then use docker
[[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
sudo -E USE_DOCKER="${use_docker:-}" ./image_builder.sh ${ROOTFS_DIR}
popd
} }
install_rootfs_image() { install_rootfs_image() {
cd ${katacontainers_repo_dir}/tools/osbuilder/image-builder pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
commit=$(git log --format=%h -1 HEAD) commit=$(git log --format=%h -1 HEAD)
date=$(date +%Y-%m-%d-%T.%N%z) date=$(date +%Y-%m-%d-%T.%N%z)
image="kata-containers-${date}-${commit}" image="kata-containers-${date}-${commit}"
@ -287,15 +312,17 @@ install_rootfs_image() {
(cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img) (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
echo "Built Rootfs from ${ROOTFS_DIR} to /usr/share/kata-containers/${image}" echo "Built Rootfs from ${ROOTFS_DIR} to /usr/share/kata-containers/${image}"
ls -al /usr/share/kata-containers/ ls -al /usr/share/kata-containers/
popd
} }
install_guest_kernel_image() { install_guest_kernel_image() {
cd ${katacontainers_repo_dir}/tools/packaging/kernel pushd ${katacontainers_repo_dir}/tools/packaging/kernel
./build-kernel.sh setup sudo -E PATH=$PATH ./build-kernel.sh setup
./build-kernel.sh build sudo -E PATH=$PATH ./build-kernel.sh build
sudo chmod 777 /usr/share/kata-containers/ # Give user permission to install kernel sudo chmod u+wrx /usr/share/kata-containers/ # Give user permission to install kernel
./build-kernel.sh install sudo -E PATH=$PATH ./build-kernel.sh install
debug_output "New kernel installed to $(ls -al /usr/share/kata-containers/vmlinux*)" debug_output "New kernel installed to $(ls -al /usr/share/kata-containers/vmlinux*)"
popd
} }
build_qemu() { build_qemu() {
@ -307,17 +334,29 @@ check_kata_runtime() {
} }
init_kubernetes() { init_kubernetes() {
# If kubernetes init has previous run we need to clean it by removing the image and resetting k8s
cid=$(docker ps -a -q -f name=^/kata-registry$) # Check that kubeadm was installed and install it otherwise
if [ -n "${cid}" ]; then if ! [ -x "$(command -v kubeadm)" ]; then
docker stop ${cid} && docker rm ${cid} pushd "${tests_repo_dir}/.ci"
fi sudo -E PATH=$PATH -s install_kubernetes.sh
k8s_nodes=$(kubectl get nodes -o name) if [ "${CRI_CONTAINERD}" == "yes" ]; then
if [ -n "${k8s_nodes}" ]; then sudo -E PATH=$PATH -s "configure_containerd_for_kubernetes.sh"
kubeadm reset -f fi
popd
fi fi
export CI="true" && ${tests_repo_dir}/integration/kubernetes/init.sh # If kubernetes init has previously run we need to clean it by removing the image and resetting k8s
cid=$(sudo docker ps -a -q -f name=^/kata-registry$)
if [ -n "${cid}" ]; then
sudo docker stop ${cid} && sudo docker rm ${cid}
fi
k8s_nodes=$(kubectl get nodes -o name 2>/dev/null || true)
if [ -n "${k8s_nodes}" ]; then
sudo kubeadm reset -f
fi
export CI="true" && sudo -E PATH=$PATH -s ${tests_repo_dir}/integration/kubernetes/init.sh
sudo chown ${USER}:$(id -g -n ${USER}) "$HOME/.kube/config"
cat << EOT | tee ~/nginx-kata.yaml cat << EOT | tee ~/nginx-kata.yaml
apiVersion: v1 apiVersion: v1
kind: Pod kind: Pod
@ -343,8 +382,8 @@ metadata:
EOF EOF
# If already exists then delete and re-create # If already exists then delete and re-create
if [ -n "$(crictl pods --name ${crictl_sandbox_name} -q)" ]; then if [ -n "$(sudo crictl pods --name ${crictl_sandbox_name} -q)" ]; then
crictl_delete_cc_pod crictl_delete_cc
fi fi
pod_id=$(sudo crictl runp -r kata ~/pod-config.yaml) pod_id=$(sudo crictl runp -r kata ~/pod-config.yaml)
@ -363,16 +402,18 @@ command:
log_path: kata-cc.0.log log_path: kata-cc.0.log
EOF EOF
pod_id=$(crictl pods --name ${crictl_sandbox_name} -q) pod_id=$(sudo crictl pods --name ${crictl_sandbox_name} -q)
container_id=$(sudo crictl create -with-pull ${pod_id} ~/container-config.yaml ~/pod-config.yaml) container_id=$(sudo crictl create -with-pull ${pod_id} ~/container-config.yaml ~/pod-config.yaml)
sudo crictl start ${container_id} sudo crictl start ${container_id}
sudo crictl ps -a sudo crictl ps -a
} }
crictl_delete_cc() { crictl_delete_cc() {
pod_id=$(crictl pods --name ${crictl_sandbox_name} -q) pod_id=$(sudo crictl pods --name ${crictl_sandbox_name} -q)
container_id=$(crictl ps --pod ${pod_id} -q) container_id=$(sudo crictl ps --pod ${pod_id} -q)
sudo crictl stop ${container_id} && sudo crictl rm ${container_id} if [ -n "${container_id}" ]; then
sudo crictl stop ${container_id} && sudo crictl rm ${container_id}
fi
sudo crictl stopp ${pod_id} && sudo crictl rmp ${pod_id} sudo crictl stopp ${pod_id} && sudo crictl rmp ${pod_id}
} }
@ -393,8 +434,11 @@ restart_kata_pod() {
test_kata_runtime() { test_kata_runtime() {
echo "Running ctr with the kata runtime..." echo "Running ctr with the kata runtime..."
test_image="docker.io/library/busybox:latest" test_image="docker.io/library/busybox:latest"
sudo ctr image pull "${test_image}" if [ -z $(ctr images ls -q name=="${test_image}") ]; then
# If you hit too many requests run `sudo ctr image pull "docker.io/library/busybox:latest" -u <dockerhub username>` command and retry sudo ctr image pull "${test_image}"
# If you hit too many requests run
# `sudo ctr image pull "docker.io/library/busybox:latest" -u <dockerhub username>` command and retry
fi
sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "${test_image}" test-kata uname -a sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "${test_image}" test-kata uname -a
} }
@ -411,7 +455,8 @@ run_kata_and_capture_logs() {
} }
get_ids() { get_ids() {
guest_cid=$(ps -ef | grep qemu-system-x86_64 | egrep -o "guest-cid=[0-9]*" | cut -d= -f2) && sandbox_id=$(ps -ef | grep qemu | egrep -o "sandbox-[^,][^,]*" | sed 's/sandbox-//g' | awk '{print $1}') guest_cid=$(sudo ss -H --vsock | awk '{print $6}' | cut -d: -f1)
sandbox_id=$(ps -ef | grep qemu | egrep -o "sandbox-[^,][^,]*" | sed 's/sandbox-//g' | awk '{print $1}')
} }
open_kata_console() { open_kata_console() {
@ -421,7 +466,7 @@ open_kata_console() {
open_kata_shell() { open_kata_shell() {
get_ids get_ids
sudo kata-runtime exec ${sandbox_id} sudo -E "PATH=$PATH" kata-runtime exec ${sandbox_id}
} }
build_bundle_dir_if_necessary() { build_bundle_dir_if_necessary() {
@ -441,8 +486,11 @@ build_agent_ctl() {
if [ -e "${HOME}/.cargo/registry" ]; then if [ -e "${HOME}/.cargo/registry" ]; then
sudo chown -R ${USER}:${USER} "${HOME}/.cargo/registry" sudo chown -R ${USER}:${USER} "${HOME}/.cargo/registry"
fi fi
make sudo -E PATH=$PATH -s make
cd "./target/x86_64-unknown-linux-musl/release" ARCH=$(uname -m)
[ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
[ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
cd "./target/${ARCH}-unknown-linux-${LIBC}/release/"
} }
run_agent_ctl_command() { run_agent_ctl_command() {
@ -466,7 +514,7 @@ agent_create_container() {
shim_pull_image() { shim_pull_image() {
get_ids get_ids
ctr_shim_command="ctr --namespace k8s.io shim --id ${sandbox_id} pull-image ${PULL_IMAGE} ${CONTAINER_ID}" ctr_shim_command="sudo ctr --namespace k8s.io shim --id ${sandbox_id} pull-image ${PULL_IMAGE} ${CONTAINER_ID}"
echo "Issuing command '${ctr_shim_command}'" echo "Issuing command '${ctr_shim_command}'"
${ctr_shim_command} ${ctr_shim_command}
} }
@ -476,6 +524,7 @@ main() {
case "$opt" in case "$opt" in
d) d)
DEBUG="-d" DEBUG="-d"
set -x
;; ;;
h) h)
usage 0 usage 0
@ -564,12 +613,12 @@ main() {
agent_pull_image) agent_pull_image)
agent_pull_image agent_pull_image
;; ;;
shim_pull_image)
shim_pull_image
;;
agent_create_container) agent_create_container)
agent_create_container agent_create_container
;; ;;
shim_pull_image)
shim_pull_image
;;
*) *)
usage 1 usage 1
;; ;;

102
docs/how-to/how-to-build-and-test-ccv0.md
View File

@ -18,8 +18,6 @@ we have working.
### Basic script set-up and optional environment variables ### Basic script set-up and optional environment variables
In order to build, configure and demo the CCv0 functionality, these are the set-up steps I take: In order to build, configure and demo the CCv0 functionality, these are the set-up steps I take:
> **Note**: I've only tested this script running as root. I hope to fix this under
[issue #2879](https://github.com/kata-containers/kata-containers/issues/2879)
- Provision a new VM - Provision a new VM
- *I choose a Ubuntu 20.04 8GB VM for this as I had one available. There are some dependences on apt-get installed* - *I choose a Ubuntu 20.04 8GB VM for this as I had one available. There are some dependences on apt-get installed*
*packages, so these will need re-working to be compatible with other platforms.* *packages, so these will need re-working to be compatible with other platforms.*
@ -45,15 +43,16 @@ $ chmod u+x ccv0.sh
$ export SKOPEO=yes $ export SKOPEO=yes
``` ```
`skopeo` is `skopeo` is
required for verifying container image signatures of pulled images. required for passing source credentials and verifying container image signatures using the kata agent.
### Using `crictl` to do end-to-end testing of provisioning a container with the unencrypted image pulled on the guest ### Using `crictl` to do end-to-end testing of provisioning a container with the unencrypted image pulled on the guest
- Run the full build process with Kubernetes off, so it's configure doesn't interfere with `crictl` using: - Run the full build process with Kubernetes off, so it's configure doesn't interfere with `crictl` using:
```bash ```bash
$ export KUBERNETES="no" $ export KUBERNETES="no"
$ . ~/ccv0.sh -d build_and_install_all $ ~/ccv0.sh -d build_and_install_all
``` ```
> **Note**: Much of this script has to be run as `sudo`, so you are likely to get prompted for your password.
- *I run this script sourced just so that the required installed components are accessible on the `PATH` to the rest* - *I run this script sourced just so that the required installed components are accessible on the `PATH` to the rest*
*of the process without having to reload the session.* *of the process without having to reload the session.*
- The steps that `build_and_install_all` takes is: - The steps that `build_and_install_all` takes is:
@ -67,19 +66,27 @@ $ chmod u+x ccv0.sh
20.04 and has extra packages like `umoci` added. 20.04 and has extra packages like `umoci` added.
- Build the Kata guest kernel - Build the Kata guest kernel
- Install QEMU - Install QEMU
> **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
this, login into Docker Hub and pull the images used with:
> ```bash
> $ docker login
> $ sudo docker pull ubuntu
> ```
> then re-run the command.
- The first time this runs it may take a while, but subsequent runs will be quicker as more things are already - The first time this runs it may take a while, but subsequent runs will be quicker as more things are already
installed and they can be further cut down by not running all the above steps installed and they can be further cut down by not running all the above steps
[see "Additional script usage" below](#additional-script-usage) [see "Additional script usage" below](#additional-script-usage)
- Create a new kata sandbox pod using `crictl` with: - Create a new kata sandbox pod using `crictl` with:
```bash ```bash
$ . ~/ccv0.sh crictl_create_cc_pod $ ~/ccv0.sh crictl_create_cc_pod
``` ```
- This creates a pod configuration file, creates the pod from this using - This creates a pod configuration file, creates the pod from this using
`sudo crictl runp -r kata ~/pod-config.yaml` and runs `sudo crictl pods` to show the pod `sudo crictl runp -r kata ~/pod-config.yaml` and runs `sudo crictl pods` to show the pod
- Create a new kata confidential container with: - Create a new kata confidential container with:
```bash ```bash
$ . ~/ccv0.sh crictl_create_cc_container $ ~/ccv0.sh crictl_create_cc_container
``` ```
- This creates a container (based on `busybox:1.33.1`) in the kata cc sandbox and prints a list of containers. - This creates a container (based on `busybox:1.33.1`) in the kata cc sandbox and prints a list of containers.
This will have been created based on an image pulled in the kata pod sandbox/guest, not on the host machine. This will have been created based on an image pulled in the kata pod sandbox/guest, not on the host machine.
@ -114,19 +121,16 @@ there.
- To verify that the image wasn't pulled on the host system we can look at the shared sandbox on the host and we - To verify that the image wasn't pulled on the host system we can look at the shared sandbox on the host and we
should only see a single bundle for the pause container as the `busybox` based container image should have been should only see a single bundle for the pause container as the `busybox` based container image should have been
pulled on the guest: pulled on the guest:
- Change to the pod's shared directory with: - Find all the `rootfs` directories under in the pod's shared directory with:
```bash ```bash
$ cd /run/kata-containers/shared/sandboxes/${pod_id}/shared $ pod_id=$(ps -ef | grep qemu | egrep -o "sandbox-[^,][^,]*" | sed 's/sandbox-//g' | awk '{print $1}')
``` $ sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs
- Find all the `rootfs` directories under this with:
```bash
$ find . -name rootfs
./e89596e9de45ef2a154a5164554c9816293ab757cfd7a53d593fa144192a9964/rootfs ./e89596e9de45ef2a154a5164554c9816293ab757cfd7a53d593fa144192a9964/rootfs
``` ```
which should only show a single `rootfs` directory if the container image was pulled on the guest, not the host which should only show a single `rootfs` directory if the container image was pulled on the guest, not the host
- Looking that `rootfs` directory with - Looking that `rootfs` directory with
```bash ```bash
$ ls -ltr ./e89596e9de45ef2a154a5164554c9816293ab757cfd7a53d593fa144192a9964/rootfs/ $ sudo ls -ltr $(sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs)
``` ```
prints something similar to prints something similar to
``` ```
@ -155,11 +159,18 @@ but Kubernetes can be used to create a non-confidential kata pod using `ccv0.sh`
as a single node cluster: as a single node cluster:
```bash ```bash
$ export KUBERNETES="yes" $ export KUBERNETES="yes"
$ . ~/ccv0.sh -d build_and_install_all $ ~/ccv0.sh build_and_install_all
``` ```
> **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
during "Store custom stress image in registry" matching `ERROR: toomanyrequests: Too Many Requests`. To get past during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
this, log into docker hub with `docker login` and re-run the step with `. ~/ccv0.sh -d init_kubernetes`. this, login into Docker Hub and pull the images used with:
> ```bash
> $ docker login
> $ sudo docker pull registry:2
> $ sudo docker pull nginx
> $ sudo docker pull ubuntu
> ```
> then re-run the command.
- Check that your Kubernetes cluster has been correctly set-up: - Check that your Kubernetes cluster has been correctly set-up:
``` ```
$ kubectl get nodes $ kubectl get nodes
@ -168,7 +179,7 @@ stevenh-ccv0-demo1.fyre.ibm.com Ready control-plane,master 3m33s v1.21.
``` ```
- Create a kata pod: - Create a kata pod:
``` ```
$ ~/ccv0.sh -d create_kata_pod $ ~/ccv0.sh create_kata_pod
pod/nginx-kata created pod/nginx-kata created
NAME READY STATUS RESTARTS AGE NAME READY STATUS RESTARTS AGE
nginx-kata 0/1 ContainerCreating 0 5s nginx-kata 0/1 ContainerCreating 0 5s
@ -196,7 +207,7 @@ that aren't broadly supported end-to-end. Here are some examples:
- For debugging purposed you can optionally create a new terminal on the VM and connect to the kata guest's console log: - For debugging purposed you can optionally create a new terminal on the VM and connect to the kata guest's console log:
```bash ```bash
$ ~/ccv0.sh -d open_kata_console $ ~/ccv0.sh open_kata_console
``` ```
- In the first terminal run the pull image on guest command against the Kata agent, via the shim (`containerd-shim-kata-v2`). - In the first terminal run the pull image on guest command against the Kata agent, via the shim (`containerd-shim-kata-v2`).
This can be achieved using the [containerd](https://github.com/containerd/containerd) CLI tool, `ctr`, which can be used to This can be achieved using the [containerd](https://github.com/containerd/containerd) CLI tool, `ctr`, which can be used to
@ -215,7 +226,7 @@ the `ccv0.sh` script to automatically fill in the variables:
below steps to run the pull image command against the agent directly. below steps to run the pull image command against the agent directly.
- Run the pull image agent endpoint with: - Run the pull image agent endpoint with:
```bash ```bash
$ ~/ccv0.sh -d shim_pull_image $ ~/ccv0.sh shim_pull_image
``` ```
which we print the `ctr shim` command for reference which we print the `ctr shim` command for reference
- Alternatively you can issue the command directly to the kata-agent pull image endpoint, which also supports - Alternatively you can issue the command directly to the kata-agent pull image endpoint, which also supports
@ -230,9 +241,12 @@ the `ccv0.sh` script to automatically fill in the variables:
proof of concept to allow more images to be pulled and tested. Once we have support for getting proof of concept to allow more images to be pulled and tested. Once we have support for getting
keys into the kata guest using the attestation-agent and/or KBS I'd expect container registry keys into the kata guest using the attestation-agent and/or KBS I'd expect container registry
credentials to be looked up using that mechanism. credentials to be looked up using that mechanism.
> **Note**: the native rust implementation doesn't current flow credentials at the moment, so use
the `skopeo` based implementation if they are needed now.
- Run the pull image agent endpoint with - Run the pull image agent endpoint with
```bash ```bash
$ ~/ccv0.sh -d agent_pull_image $ ~/ccv0.sh agent_pull_image
``` ```
and you should see output which includes `Command PullImage (1 of 1) returned (Ok(()), false)` to indicate and you should see output which includes `Command PullImage (1 of 1) returned (Ok(()), false)` to indicate
that the `PullImage` request was successful e.g. that the `PullImage` request was successful e.g.
@ -244,11 +258,11 @@ the `ccv0.sh` script to automatically fill in the variables:
{"msg":"response received","level":"INFO","ts":"2021-09-15T08:40:43.828200633-07:00","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","pid":"830920","response":""} {"msg":"response received","level":"INFO","ts":"2021-09-15T08:40:43.828200633-07:00","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","pid":"830920","response":""}
{"msg":"Command PullImage (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-15T08:40:43.828261708-07:00","subsystem":"rpc","pid":"830920","source":"kata-agent-ctl","version":"0.1.0","name":"kata-agent-ctl"} {"msg":"Command PullImage (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-15T08:40:43.828261708-07:00","subsystem":"rpc","pid":"830920","source":"kata-agent-ctl","version":"0.1.0","name":"kata-agent-ctl"}
``` ```
> **Note**: The first time that `~/ccv0.sh -d agent_pull_image` is run, the `agent-ctl` tool with be complied > **Note**: The first time that `~/ccv0.sh agent_pull_image` is run, the `agent-ctl` tool will be built
which may take a few minutes. which may take a few minutes.
- To validate that the image pull was successful, you can open a shell into the kata pod with: - To validate that the image pull was successful, you can open a shell into the kata pod with:
```bash ```bash
$ ~/ccv0.sh -d open_kata_shell $ ~/ccv0.sh open_kata_shell
``` ```
- Check the `/run/kata-containers/` directory to verify that the container image bundle has been created in a directory - Check the `/run/kata-containers/` directory to verify that the container image bundle has been created in a directory
named either `01234556789` (for the container id), or the container image name, e.g. named either `01234556789` (for the container id), or the container image name, e.g.
@ -264,9 +278,9 @@ the `ccv0.sh` script to automatically fill in the variables:
-rw-r--r-- 1 root root 63584 Jan 20 16:45 sha256_be9faa75035c20288cde7d2cdeb6cd1f5f4dbcd845d3f86f7feab61c4eff9eb5.mtree -rw-r--r-- 1 root root 63584 Jan 20 16:45 sha256_be9faa75035c20288cde7d2cdeb6cd1f5f4dbcd845d3f86f7feab61c4eff9eb5.mtree
``` ```
- Leave the kata shell by running: - Leave the kata shell by running:
```bash ```bash
$ exit $ exit
``` ```
## Verifying signed images ## Verifying signed images
@ -306,14 +320,14 @@ image by running:
```bash ```bash
$ export CONTAINER_ID="unprotected-unsigned" $ export CONTAINER_ID="unprotected-unsigned"
$ export PULL_IMAGE="docker.io/library/busybox:latest" $ export PULL_IMAGE="docker.io/library/busybox:latest"
$ ~/ccv0.sh -d agent_pull_image $ ~/ccv0.sh agent_pull_image
``` ```
- This finishes with a return `Ok()` - This finishes with a return `Ok()`
- To test that an unsigned image from our *protected* test container registry is rejected we can run: - To test that an unsigned image from our *protected* test container registry is rejected we can run:
```bash ```bash
$ export CONTAINER_ID="protected-unsigned" $ export CONTAINER_ID="protected-unsigned"
$ export PULL_IMAGE="quay.io/kata-containers/confidential-containers:unsigned" $ export PULL_IMAGE="quay.io/kata-containers/confidential-containers:unsigned"
$ ~/ccv0.sh -d agent_pull_image $ ~/ccv0.sh agent_pull_image
``` ```
- This results in an `ERROR: API failed` message from `agent_ctl` and the Kata sandbox console log shows the correct - This results in an `ERROR: API failed` message from `agent_ctl` and the Kata sandbox console log shows the correct
cause that the signature we has was not valid for the unsigned image: cause that the signature we has was not valid for the unsigned image:
@ -324,7 +338,7 @@ image by running:
```bash ```bash
$ export CONTAINER_ID="protected-signed" $ export CONTAINER_ID="protected-signed"
$ export PULL_IMAGE="quay.io/kata-containers/confidential-containers:signed" $ export PULL_IMAGE="quay.io/kata-containers/confidential-containers:signed"
$ ~/ccv0.sh -d agent_pull_image $ ~/ccv0.sh agent_pull_image
``` ```
- This finishes with a return `Ok()` - This finishes with a return `Ok()`
- Finally to check the image with a valid signature, but invalid GPG key (the real trusted piece of information we really - Finally to check the image with a valid signature, but invalid GPG key (the real trusted piece of information we really
@ -332,7 +346,7 @@ want to protect with the attestation agent in future) fails we can run:
```bash ```bash
$ export CONTAINER_ID="protected-wrong-key" $ export CONTAINER_ID="protected-wrong-key"
$ export PULL_IMAGE="quay.io/kata-containers/confidential-containers:other_signed" $ export PULL_IMAGE="quay.io/kata-containers/confidential-containers:other_signed"
$ ~/ccv0.sh -d agent_pull_image $ ~/ccv0.sh agent_pull_image
``` ```
- Again this results in an `ERROR: API failed` message from `agent_ctl` and the Kata sandbox console log shows a - Again this results in an `ERROR: API failed` message from `agent_ctl` and the Kata sandbox console log shows a
slightly different error: slightly different error:
@ -342,29 +356,31 @@ want to protect with the attestation agent in future) fails we can run:
``` ```
- To confirm that the first and third tests create the image bundles correct we can open a shell into the kata pod with: - To confirm that the first and third tests create the image bundles correct we can open a shell into the kata pod with:
```bash ```bash
$ ~/ccv0.sh -d open_kata_shell $ ~/ccv0.sh open_kata_shell
``` ```
- In the pod we can check the directories the images bundles were unpacked to: - In the pod we can check the directories the images bundles were unpacked to:
```bash ```bash
$ ls /run/kata-containers/unprotected-unsigned/ $ ls -ltr /run/kata-containers/unprotected-unsigned/
``` ```
should show something like should show something like
``` ```
config.json total 72
rootfs drwxr-xr-x 10 root root 200 Jan 1 1970 rootfs
sha256_824b88c5c38e2b931cfd471061a576a2ac1c165ef7adeae6662687031b9f9e07.mtree -rw-r--r-- 1 root root 2977 Jan 26 16:06 config.json
umoci.json -rw-r--r-- 1 root root 372 Jan 26 16:06 umoci.json
-rw-r--r-- 1 root root 63724 Jan 26 16:06 sha256_1612e16ff3f6b0d09eefdc4e9d5c5c0624f63032743e016585b095b958778016.mtree
``` ```
and and
```bash ```bash
$ ls /run/kata-containers/protected-signed/ $ ls -ltr /run/kata-containers/protected-signed/
``` ```
should show something like should show something like
``` ```
config.json total 72
rootfs drwxr-xr-x 10 root root 200 Jan 1 1970 rootfs
sha256_ebf391d3f0ba36d4b64999ebbeadc878d229faec8839254a1c2264cf47735841.mtree -rw-r--r-- 1 root root 2977 Jan 26 16:07 config.json
umoci.json -rw-r--r-- 1 root root 372 Jan 26 16:07 umoci.json
-rw-r--r-- 1 root root 63568 Jan 26 16:07 sha256_ebf391d3f0ba36d4b64999ebbeadc878d229faec8839254a1c2264cf47735841.mtree
``` ```
## Additional script usage ## Additional script usage
@ -372,9 +388,9 @@ want to protect with the attestation agent in future) fails we can run:
As well as being able to use the script as above to build all of `kata-containers` from scratch it can be used to just As well as being able to use the script as above to build all of `kata-containers` from scratch it can be used to just
re-build bits of it by running the script with different parameters. For example after the first build you will often re-build bits of it by running the script with different parameters. For example after the first build you will often
not need to re-install the dependencies, QEMU or the Guest kernel, but just test code changes made to the runtime and not need to re-install the dependencies, QEMU or the Guest kernel, but just test code changes made to the runtime and
agent. This can be done by running `. ~/ccv0.sh -d rebuild_and_install_kata`. (*Note this does a hard checkout* agent. This can be done by running `~/ccv0.sh rebuild_and_install_kata`. (*Note this does a hard checkout*
*from git, so if your changes are only made locally it is better to do the individual steps e.g.* *from git, so if your changes are only made locally it is better to do the individual steps e.g.*
`. ~/ccv0.sh -d build_kata_runtime && . ~/ccv0.sh -d build_and_add_agent_to_rootfs && . ~/ccv0.sh -d build_and_install_rootfs`). `~/ccv0.sh build_kata_runtime && ~/ccv0.sh build_and_add_agent_to_rootfs && ~/ccv0.sh build_and_install_rootfs`).
There are commands for a lot of steps in building, setting up and testing and the full list can be seen by running There are commands for a lot of steps in building, setting up and testing and the full list can be seen by running
`~/ccv0.sh help`: `~/ccv0.sh help`:
``` ```