mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-08-21 09:25:19 +00:00
Merge pull request #3564 from stevenhorsman/ccv0-improvements
CCv0 | doc: Improve ccv0.sh and CCv0 how-to document
This commit is contained in:
Steve Horsman
GitHub
commit
e71592d5da
@ -1,6 +1,6 @@
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# Copyright (c) 2021 IBM Corporation
|
||||
# Copyright (c) 2021, 2022 IBM Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
@ -8,7 +8,8 @@
|
||||
# Disclaimer: This script is work in progress for supporting the CCv0 prototype
|
||||
# It shouldn't be considered supported by the Kata Containers community, or anyone else
|
||||
|
||||
# Based on https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md, but with elements of the tests/.ci scripts used
|
||||
# Based on https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md,
|
||||
# but with elements of the tests/.ci scripts used
|
||||
|
||||
readonly script_name="$(basename "${BASH_SOURCE[0]}")"
|
||||
|
||||
@ -31,8 +32,14 @@ export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
|
||||
export tests_branch="${tests_branch:-CCv0}"
|
||||
export target_branch=${tests_branch} # kata-containers/ci/lib.sh uses target branch var to check out tests repo
|
||||
|
||||
# if .bash_profile exists then use it, otherwise fall back to .profile
|
||||
export PROFILE="${HOME}/.profile"
|
||||
if [ -r "${HOME}/.bash_profile" ]; then
|
||||
export PROFILE="${HOME}/.bash_profile"
|
||||
fi
|
||||
|
||||
# Create a bunch of common, derived values up front so we don't need to create them in all the different functions
|
||||
. "$HOME/.profile"
|
||||
. ${PROFILE}
|
||||
if [ -z ${GOPATH} ]; then
|
||||
export GOPATH=${HOME}/go
|
||||
fi
|
||||
@ -41,6 +48,8 @@ export katacontainers_repo_dir="${GOPATH}/src/${katacontainers_repo}"
|
||||
export ROOTFS_DIR="${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder/rootfs"
|
||||
export PULL_IMAGE="${PULL_IMAGE:-quay.io/kata-containers/confidential-containers:signed}" # Doesn't need authentication
|
||||
export CONTAINER_ID="${CONTAINER_ID:-0123456789}"
|
||||
source /etc/os-release || source /usr/lib/os-release
|
||||
grep -Eq "\<fedora\>" /etc/os-release 2> /dev/null && export USE_PODMAN=true
|
||||
|
||||
debug_output() {
|
||||
if [ -n "${DEBUG}" ]
|
||||
@ -65,7 +74,7 @@ usage() {
|
||||
Overview:
|
||||
Build and test kata containers from source
|
||||
Optionally set kata-containers and tests repo and branch as exported variables before running
|
||||
e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && . ~/${script_name} -d build_and_install_all
|
||||
e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/${script_name} build_and_install_all
|
||||
Usage:
|
||||
${script_name} [options] <command>
|
||||
Commands:
|
||||
@ -129,26 +138,30 @@ rebuild_and_install_kata() {
|
||||
|
||||
# Based on the jenkins_job_build.sh script in kata-containers/tests/.ci - checks out source code and installs dependencies
|
||||
initialize() {
|
||||
# We need git to checkout and bootstrap the ci scripts
|
||||
sudo apt-get update && sudo apt-get install -y git socat qemu-utils
|
||||
# We need git to checkout and bootstrap the ci scripts and some other packages used in testing
|
||||
sudo apt-get update && sudo apt-get install -y curl git socat qemu-utils
|
||||
|
||||
PROFILE="${HOME}/.profile"
|
||||
grep -qxF "export GOPATH=\${HOME}/go" "${PROFILE}" || echo "export GOPATH=\${HOME}/go" >> "${PROFILE}"
|
||||
grep -qxF "export GOROOT=/usr/local/go" "${PROFILE}" || echo "export GOROOT=/usr/local/go" >> "${PROFILE}"
|
||||
grep -qxF "export PATH=\${GOPATH}/bin:/usr/local/go/bin:/usr/sbin:/sbin:\${PATH}" "${PROFILE}" || echo "export PATH=\${GOPATH}/bin:/usr/local/go/bin:/usr/sbin:/sbin:\${PATH}" >> "${PROFILE}"
|
||||
. "${HOME}/.profile"
|
||||
grep -qxF "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" "${PROFILE}" || echo "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" >> "${PROFILE}"
|
||||
|
||||
# Load the new go and PATH parameters from the profile
|
||||
. ${PROFILE}
|
||||
mkdir -p "${GOPATH}"
|
||||
|
||||
check_out_repos
|
||||
|
||||
pushd "${tests_repo_dir}"
|
||||
ci_dir_name=".ci"
|
||||
"${ci_dir_name}/install_go.sh" -p -f
|
||||
"${ci_dir_name}/install_rust.sh"
|
||||
sudo -E PATH=$PATH -s "${ci_dir_name}/install_go.sh" -p -f
|
||||
sudo -E PATH=$PATH -s "${ci_dir_name}/install_rust.sh"
|
||||
|
||||
# Run setup, but don't install kata as we will build it ourselves in locations matching the developer guide
|
||||
export INSTALL_KATA="no"
|
||||
${ci_dir_name}/setup.sh
|
||||
sudo -E PATH=$PATH -s ${ci_dir_name}/setup.sh
|
||||
# Reload the profile to pick up installed dependencies
|
||||
. ${PROFILE}
|
||||
popd
|
||||
}
|
||||
|
||||
@ -177,11 +190,12 @@ check_out_repos() {
|
||||
}
|
||||
|
||||
build_and_install_kata_runtime() {
|
||||
cd ${katacontainers_repo_dir}/src/runtime
|
||||
pushd ${katacontainers_repo_dir}/src/runtime
|
||||
make clean && make && sudo -E PATH=$PATH make install
|
||||
debug_output "We should have created Kata runtime binaries:: /usr/local/bin/kata-runtime and /usr/local/bin/containerd-shim-kata-v2"
|
||||
debug_output "We should have made the Kata configuration file: /usr/share/defaults/kata-containers/configuration.toml"
|
||||
debug_output "kata-runtime version: $(kata-runtime version)"
|
||||
popd
|
||||
}
|
||||
|
||||
configure() {
|
||||
@ -199,7 +213,7 @@ configure() {
|
||||
# Add cni directory to containerd config
|
||||
echo " [plugins.cri.cni]
|
||||
# conf_dir is the directory in which the admin places a CNI conf.
|
||||
conf_dir = \"/etc/cni/net.d\"" >> /etc/containerd/config.toml
|
||||
conf_dir = \"/etc/cni/net.d\"" | sudo tee -a /etc/containerd/config.toml
|
||||
|
||||
# Switch image offload to true in kata config
|
||||
sudo sed -i -e 's/^# *\(service_offload\).*=.*$/\1 = true/g' /etc/kata-containers/configuration.toml
|
||||
@ -232,40 +246,48 @@ build_and_add_agent_to_rootfs() {
|
||||
}
|
||||
|
||||
build_a_custom_kata_agent() {
|
||||
if [[ ! -L /bin/musl-g++ ]]
|
||||
then
|
||||
rustup target add x86_64-unknown-linux-musl
|
||||
sudo ln -s /usr/bin/g++ /bin/musl-g++
|
||||
fi
|
||||
. "$HOME/.cargo/env"
|
||||
cd ${katacontainers_repo_dir}/src/agent && make
|
||||
debug_output "Kata agent built: $(ls -al ${katacontainers_repo_dir}/src/agent/target/x86_64-unknown-linux-musl/release/kata-agent)"
|
||||
pushd ${katacontainers_repo_dir}/src/agent
|
||||
sudo -E PATH=$PATH make
|
||||
|
||||
ARCH=$(uname -m)
|
||||
[ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
|
||||
[ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
|
||||
|
||||
debug_output "Kata agent built: $(ls -al ${katacontainers_repo_dir}/src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent)"
|
||||
# Run a make install into the rootfs directory in order to create the kata-agent.service file which is required when we add to the rootfs
|
||||
sudo -E PATH=$PATH make install DESTDIR="${ROOTFS}"
|
||||
popd
|
||||
}
|
||||
|
||||
create_a_local_rootfs() {
|
||||
sudo rm -rf "${ROOTFS_DIR}"
|
||||
cd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
|
||||
pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
|
||||
export distro="ubuntu"
|
||||
[[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
|
||||
sudo -E OS_VERSION="${OS_VERSION:-}" GOPATH=$GOPATH DEBUG="${DEBUG}" USE_DOCKER="${use_docker:-}" SKOPEO=${SKOPEO:-} UMOCI=yes SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro}
|
||||
sudo -E OS_VERSION="${OS_VERSION:-}" GOPATH=$GOPATH EXTRA_PKGS="vim iputils-ping net-tools" DEBUG="${DEBUG}" USE_DOCKER="${use_docker:-}" SKOPEO=${SKOPEO:-} UMOCI=yes SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro}
|
||||
|
||||
# Install_rust.sh during rootfs.sh switches us to the main branch of the tests repo, so switch back now
|
||||
pushd "${tests_repo_dir}"
|
||||
git checkout ${tests_branch}
|
||||
popd
|
||||
|
||||
# During the ./rootfs.sh call the kata agent is built as root, so we need to update the permissions, so we can rebuild it
|
||||
sudo chown -R ${USER}:${USER} "${katacontainers_repo_dir}/src/agent/"
|
||||
popd
|
||||
}
|
||||
|
||||
add_custom_agent_to_rootfs() {
|
||||
cd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
|
||||
sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent
|
||||
pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
|
||||
|
||||
ARCH=$(uname -m)
|
||||
[ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
|
||||
[ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
|
||||
|
||||
sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ${katacontainers_repo_dir}/src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent
|
||||
sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
|
||||
sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
|
||||
debug_output "Added kata agent to rootfs: $(ls -al ${ROOTFS_DIR}/usr/bin/kata-agent)"
|
||||
popd
|
||||
}
|
||||
|
||||
build_and_install_rootfs() {
|
||||
@ -274,12 +296,15 @@ build_and_install_rootfs() {
|
||||
}
|
||||
|
||||
build_rootfs_image() {
|
||||
cd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
|
||||
script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}'
|
||||
pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
|
||||
# Logic from install_kata_image.sh - if we aren't using podman (ie on a fedora like), then use docker
|
||||
[[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
|
||||
sudo -E USE_DOCKER="${use_docker:-}" ./image_builder.sh ${ROOTFS_DIR}
|
||||
popd
|
||||
}
|
||||
|
||||
install_rootfs_image() {
|
||||
cd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
|
||||
pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
|
||||
commit=$(git log --format=%h -1 HEAD)
|
||||
date=$(date +%Y-%m-%d-%T.%N%z)
|
||||
image="kata-containers-${date}-${commit}"
|
||||
@ -287,15 +312,17 @@ install_rootfs_image() {
|
||||
(cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
|
||||
echo "Built Rootfs from ${ROOTFS_DIR} to /usr/share/kata-containers/${image}"
|
||||
ls -al /usr/share/kata-containers/
|
||||
popd
|
||||
}
|
||||
|
||||
install_guest_kernel_image() {
|
||||
cd ${katacontainers_repo_dir}/tools/packaging/kernel
|
||||
./build-kernel.sh setup
|
||||
./build-kernel.sh build
|
||||
sudo chmod 777 /usr/share/kata-containers/ # Give user permission to install kernel
|
||||
./build-kernel.sh install
|
||||
pushd ${katacontainers_repo_dir}/tools/packaging/kernel
|
||||
sudo -E PATH=$PATH ./build-kernel.sh setup
|
||||
sudo -E PATH=$PATH ./build-kernel.sh build
|
||||
sudo chmod u+wrx /usr/share/kata-containers/ # Give user permission to install kernel
|
||||
sudo -E PATH=$PATH ./build-kernel.sh install
|
||||
debug_output "New kernel installed to $(ls -al /usr/share/kata-containers/vmlinux*)"
|
||||
popd
|
||||
}
|
||||
|
||||
build_qemu() {
|
||||
@ -307,17 +334,29 @@ check_kata_runtime() {
|
||||
}
|
||||
|
||||
init_kubernetes() {
|
||||
# If kubernetes init has previous run we need to clean it by removing the image and resetting k8s
|
||||
cid=$(docker ps -a -q -f name=^/kata-registry$)
|
||||
if [ -n "${cid}" ]; then
|
||||
docker stop ${cid} && docker rm ${cid}
|
||||
fi
|
||||
k8s_nodes=$(kubectl get nodes -o name)
|
||||
if [ -n "${k8s_nodes}" ]; then
|
||||
kubeadm reset -f
|
||||
|
||||
# Check that kubeadm was installed and install it otherwise
|
||||
if ! [ -x "$(command -v kubeadm)" ]; then
|
||||
pushd "${tests_repo_dir}/.ci"
|
||||
sudo -E PATH=$PATH -s install_kubernetes.sh
|
||||
if [ "${CRI_CONTAINERD}" == "yes" ]; then
|
||||
sudo -E PATH=$PATH -s "configure_containerd_for_kubernetes.sh"
|
||||
fi
|
||||
popd
|
||||
fi
|
||||
|
||||
export CI="true" && ${tests_repo_dir}/integration/kubernetes/init.sh
|
||||
# If kubernetes init has previously run we need to clean it by removing the image and resetting k8s
|
||||
cid=$(sudo docker ps -a -q -f name=^/kata-registry$)
|
||||
if [ -n "${cid}" ]; then
|
||||
sudo docker stop ${cid} && sudo docker rm ${cid}
|
||||
fi
|
||||
k8s_nodes=$(kubectl get nodes -o name 2>/dev/null || true)
|
||||
if [ -n "${k8s_nodes}" ]; then
|
||||
sudo kubeadm reset -f
|
||||
fi
|
||||
|
||||
export CI="true" && sudo -E PATH=$PATH -s ${tests_repo_dir}/integration/kubernetes/init.sh
|
||||
sudo chown ${USER}:$(id -g -n ${USER}) "$HOME/.kube/config"
|
||||
cat << EOT | tee ~/nginx-kata.yaml
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
@ -343,8 +382,8 @@ metadata:
|
||||
EOF
|
||||
|
||||
# If already exists then delete and re-create
|
||||
if [ -n "$(crictl pods --name ${crictl_sandbox_name} -q)" ]; then
|
||||
crictl_delete_cc_pod
|
||||
if [ -n "$(sudo crictl pods --name ${crictl_sandbox_name} -q)" ]; then
|
||||
crictl_delete_cc
|
||||
fi
|
||||
|
||||
pod_id=$(sudo crictl runp -r kata ~/pod-config.yaml)
|
||||
@ -363,16 +402,18 @@ command:
|
||||
log_path: kata-cc.0.log
|
||||
EOF
|
||||
|
||||
pod_id=$(crictl pods --name ${crictl_sandbox_name} -q)
|
||||
pod_id=$(sudo crictl pods --name ${crictl_sandbox_name} -q)
|
||||
container_id=$(sudo crictl create -with-pull ${pod_id} ~/container-config.yaml ~/pod-config.yaml)
|
||||
sudo crictl start ${container_id}
|
||||
sudo crictl ps -a
|
||||
}
|
||||
|
||||
crictl_delete_cc() {
|
||||
pod_id=$(crictl pods --name ${crictl_sandbox_name} -q)
|
||||
container_id=$(crictl ps --pod ${pod_id} -q)
|
||||
sudo crictl stop ${container_id} && sudo crictl rm ${container_id}
|
||||
pod_id=$(sudo crictl pods --name ${crictl_sandbox_name} -q)
|
||||
container_id=$(sudo crictl ps --pod ${pod_id} -q)
|
||||
if [ -n "${container_id}" ]; then
|
||||
sudo crictl stop ${container_id} && sudo crictl rm ${container_id}
|
||||
fi
|
||||
sudo crictl stopp ${pod_id} && sudo crictl rmp ${pod_id}
|
||||
}
|
||||
|
||||
@ -393,8 +434,11 @@ restart_kata_pod() {
|
||||
test_kata_runtime() {
|
||||
echo "Running ctr with the kata runtime..."
|
||||
test_image="docker.io/library/busybox:latest"
|
||||
sudo ctr image pull "${test_image}"
|
||||
# If you hit too many requests run `sudo ctr image pull "docker.io/library/busybox:latest" -u <dockerhub username>` command and retry
|
||||
if [ -z $(ctr images ls -q name=="${test_image}") ]; then
|
||||
sudo ctr image pull "${test_image}"
|
||||
# If you hit too many requests run
|
||||
# `sudo ctr image pull "docker.io/library/busybox:latest" -u <dockerhub username>` command and retry
|
||||
fi
|
||||
sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "${test_image}" test-kata uname -a
|
||||
}
|
||||
|
||||
@ -411,7 +455,8 @@ run_kata_and_capture_logs() {
|
||||
}
|
||||
|
||||
get_ids() {
|
||||
guest_cid=$(ps -ef | grep qemu-system-x86_64 | egrep -o "guest-cid=[0-9]*" | cut -d= -f2) && sandbox_id=$(ps -ef | grep qemu | egrep -o "sandbox-[^,][^,]*" | sed 's/sandbox-//g' | awk '{print $1}')
|
||||
guest_cid=$(sudo ss -H --vsock | awk '{print $6}' | cut -d: -f1)
|
||||
sandbox_id=$(ps -ef | grep qemu | egrep -o "sandbox-[^,][^,]*" | sed 's/sandbox-//g' | awk '{print $1}')
|
||||
}
|
||||
|
||||
open_kata_console() {
|
||||
@ -421,7 +466,7 @@ open_kata_console() {
|
||||
|
||||
open_kata_shell() {
|
||||
get_ids
|
||||
sudo kata-runtime exec ${sandbox_id}
|
||||
sudo -E "PATH=$PATH" kata-runtime exec ${sandbox_id}
|
||||
}
|
||||
|
||||
build_bundle_dir_if_necessary() {
|
||||
@ -441,8 +486,11 @@ build_agent_ctl() {
|
||||
if [ -e "${HOME}/.cargo/registry" ]; then
|
||||
sudo chown -R ${USER}:${USER} "${HOME}/.cargo/registry"
|
||||
fi
|
||||
make
|
||||
cd "./target/x86_64-unknown-linux-musl/release"
|
||||
sudo -E PATH=$PATH -s make
|
||||
ARCH=$(uname -m)
|
||||
[ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
|
||||
[ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
|
||||
cd "./target/${ARCH}-unknown-linux-${LIBC}/release/"
|
||||
}
|
||||
|
||||
run_agent_ctl_command() {
|
||||
@ -466,7 +514,7 @@ agent_create_container() {
|
||||
|
||||
shim_pull_image() {
|
||||
get_ids
|
||||
ctr_shim_command="ctr --namespace k8s.io shim --id ${sandbox_id} pull-image ${PULL_IMAGE} ${CONTAINER_ID}"
|
||||
ctr_shim_command="sudo ctr --namespace k8s.io shim --id ${sandbox_id} pull-image ${PULL_IMAGE} ${CONTAINER_ID}"
|
||||
echo "Issuing command '${ctr_shim_command}'"
|
||||
${ctr_shim_command}
|
||||
}
|
||||
@ -476,6 +524,7 @@ main() {
|
||||
case "$opt" in
|
||||
d)
|
||||
DEBUG="-d"
|
||||
set -x
|
||||
;;
|
||||
h)
|
||||
usage 0
|
||||
@ -564,12 +613,12 @@ main() {
|
||||
agent_pull_image)
|
||||
agent_pull_image
|
||||
;;
|
||||
shim_pull_image)
|
||||
shim_pull_image
|
||||
;;
|
||||
agent_create_container)
|
||||
agent_create_container
|
||||
;;
|
||||
shim_pull_image)
|
||||
shim_pull_image
|
||||
;;
|
||||
*)
|
||||
usage 1
|
||||
;;
|
||||
|
||||
@ -18,8 +18,6 @@ we have working.
|
||||
### Basic script set-up and optional environment variables
|
||||
|
||||
In order to build, configure and demo the CCv0 functionality, these are the set-up steps I take:
|
||||
> **Note**: I've only tested this script running as root. I hope to fix this under
|
||||
[issue #2879](https://github.com/kata-containers/kata-containers/issues/2879)
|
||||
- Provision a new VM
|
||||
- *I choose a Ubuntu 20.04 8GB VM for this as I had one available. There are some dependences on apt-get installed*
|
||||
*packages, so these will need re-working to be compatible with other platforms.*
|
||||
@ -45,15 +43,16 @@ $ chmod u+x ccv0.sh
|
||||
$ export SKOPEO=yes
|
||||
```
|
||||
`skopeo` is
|
||||
required for verifying container image signatures of pulled images.
|
||||
required for passing source credentials and verifying container image signatures using the kata agent.
|
||||
|
||||
### Using `crictl` to do end-to-end testing of provisioning a container with the unencrypted image pulled on the guest
|
||||
|
||||
- Run the full build process with Kubernetes off, so it's configure doesn't interfere with `crictl` using:
|
||||
```bash
|
||||
$ export KUBERNETES="no"
|
||||
$ . ~/ccv0.sh -d build_and_install_all
|
||||
$ ~/ccv0.sh -d build_and_install_all
|
||||
```
|
||||
> **Note**: Much of this script has to be run as `sudo`, so you are likely to get prompted for your password.
|
||||
- *I run this script sourced just so that the required installed components are accessible on the `PATH` to the rest*
|
||||
*of the process without having to reload the session.*
|
||||
- The steps that `build_and_install_all` takes is:
|
||||
@ -67,19 +66,27 @@ $ chmod u+x ccv0.sh
|
||||
20.04 and has extra packages like `umoci` added.
|
||||
- Build the Kata guest kernel
|
||||
- Install QEMU
|
||||
> **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
|
||||
during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
|
||||
this, login into Docker Hub and pull the images used with:
|
||||
> ```bash
|
||||
> $ docker login
|
||||
> $ sudo docker pull ubuntu
|
||||
> ```
|
||||
> then re-run the command.
|
||||
- The first time this runs it may take a while, but subsequent runs will be quicker as more things are already
|
||||
installed and they can be further cut down by not running all the above steps
|
||||
[see "Additional script usage" below](#additional-script-usage)
|
||||
|
||||
- Create a new kata sandbox pod using `crictl` with:
|
||||
```bash
|
||||
$ . ~/ccv0.sh crictl_create_cc_pod
|
||||
$ ~/ccv0.sh crictl_create_cc_pod
|
||||
```
|
||||
- This creates a pod configuration file, creates the pod from this using
|
||||
`sudo crictl runp -r kata ~/pod-config.yaml` and runs `sudo crictl pods` to show the pod
|
||||
- Create a new kata confidential container with:
|
||||
```bash
|
||||
$ . ~/ccv0.sh crictl_create_cc_container
|
||||
$ ~/ccv0.sh crictl_create_cc_container
|
||||
```
|
||||
- This creates a container (based on `busybox:1.33.1`) in the kata cc sandbox and prints a list of containers.
|
||||
This will have been created based on an image pulled in the kata pod sandbox/guest, not on the host machine.
|
||||
@ -114,19 +121,16 @@ there.
|
||||
- To verify that the image wasn't pulled on the host system we can look at the shared sandbox on the host and we
|
||||
should only see a single bundle for the pause container as the `busybox` based container image should have been
|
||||
pulled on the guest:
|
||||
- Change to the pod's shared directory with:
|
||||
- Find all the `rootfs` directories under in the pod's shared directory with:
|
||||
```bash
|
||||
$ cd /run/kata-containers/shared/sandboxes/${pod_id}/shared
|
||||
```
|
||||
- Find all the `rootfs` directories under this with:
|
||||
```bash
|
||||
$ find . -name rootfs
|
||||
$ pod_id=$(ps -ef | grep qemu | egrep -o "sandbox-[^,][^,]*" | sed 's/sandbox-//g' | awk '{print $1}')
|
||||
$ sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs
|
||||
./e89596e9de45ef2a154a5164554c9816293ab757cfd7a53d593fa144192a9964/rootfs
|
||||
```
|
||||
which should only show a single `rootfs` directory if the container image was pulled on the guest, not the host
|
||||
- Looking that `rootfs` directory with
|
||||
```bash
|
||||
$ ls -ltr ./e89596e9de45ef2a154a5164554c9816293ab757cfd7a53d593fa144192a9964/rootfs/
|
||||
$ sudo ls -ltr $(sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs)
|
||||
```
|
||||
prints something similar to
|
||||
```
|
||||
@ -155,11 +159,18 @@ but Kubernetes can be used to create a non-confidential kata pod using `ccv0.sh`
|
||||
as a single node cluster:
|
||||
```bash
|
||||
$ export KUBERNETES="yes"
|
||||
$ . ~/ccv0.sh -d build_and_install_all
|
||||
$ ~/ccv0.sh build_and_install_all
|
||||
```
|
||||
> **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
|
||||
during "Store custom stress image in registry" matching `ERROR: toomanyrequests: Too Many Requests`. To get past
|
||||
this, log into docker hub with `docker login` and re-run the step with `. ~/ccv0.sh -d init_kubernetes`.
|
||||
> **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
|
||||
during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
|
||||
this, login into Docker Hub and pull the images used with:
|
||||
> ```bash
|
||||
> $ docker login
|
||||
> $ sudo docker pull registry:2
|
||||
> $ sudo docker pull nginx
|
||||
> $ sudo docker pull ubuntu
|
||||
> ```
|
||||
> then re-run the command.
|
||||
- Check that your Kubernetes cluster has been correctly set-up:
|
||||
```
|
||||
$ kubectl get nodes
|
||||
@ -168,7 +179,7 @@ stevenh-ccv0-demo1.fyre.ibm.com Ready control-plane,master 3m33s v1.21.
|
||||
```
|
||||
- Create a kata pod:
|
||||
```
|
||||
$ ~/ccv0.sh -d create_kata_pod
|
||||
$ ~/ccv0.sh create_kata_pod
|
||||
pod/nginx-kata created
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
nginx-kata 0/1 ContainerCreating 0 5s
|
||||
@ -196,7 +207,7 @@ that aren't broadly supported end-to-end. Here are some examples:
|
||||
|
||||
- For debugging purposed you can optionally create a new terminal on the VM and connect to the kata guest's console log:
|
||||
```bash
|
||||
$ ~/ccv0.sh -d open_kata_console
|
||||
$ ~/ccv0.sh open_kata_console
|
||||
```
|
||||
- In the first terminal run the pull image on guest command against the Kata agent, via the shim (`containerd-shim-kata-v2`).
|
||||
This can be achieved using the [containerd](https://github.com/containerd/containerd) CLI tool, `ctr`, which can be used to
|
||||
@ -215,7 +226,7 @@ the `ccv0.sh` script to automatically fill in the variables:
|
||||
below steps to run the pull image command against the agent directly.
|
||||
- Run the pull image agent endpoint with:
|
||||
```bash
|
||||
$ ~/ccv0.sh -d shim_pull_image
|
||||
$ ~/ccv0.sh shim_pull_image
|
||||
```
|
||||
which we print the `ctr shim` command for reference
|
||||
- Alternatively you can issue the command directly to the kata-agent pull image endpoint, which also supports
|
||||
@ -230,9 +241,12 @@ the `ccv0.sh` script to automatically fill in the variables:
|
||||
proof of concept to allow more images to be pulled and tested. Once we have support for getting
|
||||
keys into the kata guest using the attestation-agent and/or KBS I'd expect container registry
|
||||
credentials to be looked up using that mechanism.
|
||||
|
||||
> **Note**: the native rust implementation doesn't current flow credentials at the moment, so use
|
||||
the `skopeo` based implementation if they are needed now.
|
||||
- Run the pull image agent endpoint with
|
||||
```bash
|
||||
$ ~/ccv0.sh -d agent_pull_image
|
||||
$ ~/ccv0.sh agent_pull_image
|
||||
```
|
||||
and you should see output which includes `Command PullImage (1 of 1) returned (Ok(()), false)` to indicate
|
||||
that the `PullImage` request was successful e.g.
|
||||
@ -244,11 +258,11 @@ the `ccv0.sh` script to automatically fill in the variables:
|
||||
{"msg":"response received","level":"INFO","ts":"2021-09-15T08:40:43.828200633-07:00","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","pid":"830920","response":""}
|
||||
{"msg":"Command PullImage (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-15T08:40:43.828261708-07:00","subsystem":"rpc","pid":"830920","source":"kata-agent-ctl","version":"0.1.0","name":"kata-agent-ctl"}
|
||||
```
|
||||
> **Note**: The first time that `~/ccv0.sh -d agent_pull_image` is run, the `agent-ctl` tool with be complied
|
||||
> **Note**: The first time that `~/ccv0.sh agent_pull_image` is run, the `agent-ctl` tool will be built
|
||||
which may take a few minutes.
|
||||
- To validate that the image pull was successful, you can open a shell into the kata pod with:
|
||||
```bash
|
||||
$ ~/ccv0.sh -d open_kata_shell
|
||||
$ ~/ccv0.sh open_kata_shell
|
||||
```
|
||||
- Check the `/run/kata-containers/` directory to verify that the container image bundle has been created in a directory
|
||||
named either `01234556789` (for the container id), or the container image name, e.g.
|
||||
@ -264,9 +278,9 @@ the `ccv0.sh` script to automatically fill in the variables:
|
||||
-rw-r--r-- 1 root root 63584 Jan 20 16:45 sha256_be9faa75035c20288cde7d2cdeb6cd1f5f4dbcd845d3f86f7feab61c4eff9eb5.mtree
|
||||
```
|
||||
- Leave the kata shell by running:
|
||||
```bash
|
||||
$ exit
|
||||
```
|
||||
```bash
|
||||
$ exit
|
||||
```
|
||||
|
||||
## Verifying signed images
|
||||
|
||||
@ -306,14 +320,14 @@ image by running:
|
||||
```bash
|
||||
$ export CONTAINER_ID="unprotected-unsigned"
|
||||
$ export PULL_IMAGE="docker.io/library/busybox:latest"
|
||||
$ ~/ccv0.sh -d agent_pull_image
|
||||
$ ~/ccv0.sh agent_pull_image
|
||||
```
|
||||
- This finishes with a return `Ok()`
|
||||
- To test that an unsigned image from our *protected* test container registry is rejected we can run:
|
||||
```bash
|
||||
$ export CONTAINER_ID="protected-unsigned"
|
||||
$ export PULL_IMAGE="quay.io/kata-containers/confidential-containers:unsigned"
|
||||
$ ~/ccv0.sh -d agent_pull_image
|
||||
$ ~/ccv0.sh agent_pull_image
|
||||
```
|
||||
- This results in an `ERROR: API failed` message from `agent_ctl` and the Kata sandbox console log shows the correct
|
||||
cause that the signature we has was not valid for the unsigned image:
|
||||
@ -324,7 +338,7 @@ image by running:
|
||||
```bash
|
||||
$ export CONTAINER_ID="protected-signed"
|
||||
$ export PULL_IMAGE="quay.io/kata-containers/confidential-containers:signed"
|
||||
$ ~/ccv0.sh -d agent_pull_image
|
||||
$ ~/ccv0.sh agent_pull_image
|
||||
```
|
||||
- This finishes with a return `Ok()`
|
||||
- Finally to check the image with a valid signature, but invalid GPG key (the real trusted piece of information we really
|
||||
@ -332,7 +346,7 @@ want to protect with the attestation agent in future) fails we can run:
|
||||
```bash
|
||||
$ export CONTAINER_ID="protected-wrong-key"
|
||||
$ export PULL_IMAGE="quay.io/kata-containers/confidential-containers:other_signed"
|
||||
$ ~/ccv0.sh -d agent_pull_image
|
||||
$ ~/ccv0.sh agent_pull_image
|
||||
```
|
||||
- Again this results in an `ERROR: API failed` message from `agent_ctl` and the Kata sandbox console log shows a
|
||||
slightly different error:
|
||||
@ -342,29 +356,31 @@ want to protect with the attestation agent in future) fails we can run:
|
||||
```
|
||||
- To confirm that the first and third tests create the image bundles correct we can open a shell into the kata pod with:
|
||||
```bash
|
||||
$ ~/ccv0.sh -d open_kata_shell
|
||||
$ ~/ccv0.sh open_kata_shell
|
||||
```
|
||||
- In the pod we can check the directories the images bundles were unpacked to:
|
||||
```bash
|
||||
$ ls /run/kata-containers/unprotected-unsigned/
|
||||
$ ls -ltr /run/kata-containers/unprotected-unsigned/
|
||||
```
|
||||
should show something like
|
||||
```
|
||||
config.json
|
||||
rootfs
|
||||
sha256_824b88c5c38e2b931cfd471061a576a2ac1c165ef7adeae6662687031b9f9e07.mtree
|
||||
umoci.json
|
||||
total 72
|
||||
drwxr-xr-x 10 root root 200 Jan 1 1970 rootfs
|
||||
-rw-r--r-- 1 root root 2977 Jan 26 16:06 config.json
|
||||
-rw-r--r-- 1 root root 372 Jan 26 16:06 umoci.json
|
||||
-rw-r--r-- 1 root root 63724 Jan 26 16:06 sha256_1612e16ff3f6b0d09eefdc4e9d5c5c0624f63032743e016585b095b958778016.mtree
|
||||
```
|
||||
and
|
||||
```bash
|
||||
$ ls /run/kata-containers/protected-signed/
|
||||
$ ls -ltr /run/kata-containers/protected-signed/
|
||||
```
|
||||
should show something like
|
||||
```
|
||||
config.json
|
||||
rootfs
|
||||
sha256_ebf391d3f0ba36d4b64999ebbeadc878d229faec8839254a1c2264cf47735841.mtree
|
||||
umoci.json
|
||||
total 72
|
||||
drwxr-xr-x 10 root root 200 Jan 1 1970 rootfs
|
||||
-rw-r--r-- 1 root root 2977 Jan 26 16:07 config.json
|
||||
-rw-r--r-- 1 root root 372 Jan 26 16:07 umoci.json
|
||||
-rw-r--r-- 1 root root 63568 Jan 26 16:07 sha256_ebf391d3f0ba36d4b64999ebbeadc878d229faec8839254a1c2264cf47735841.mtree
|
||||
```
|
||||
|
||||
## Additional script usage
|
||||
@ -372,9 +388,9 @@ want to protect with the attestation agent in future) fails we can run:
|
||||
As well as being able to use the script as above to build all of `kata-containers` from scratch it can be used to just
|
||||
re-build bits of it by running the script with different parameters. For example after the first build you will often
|
||||
not need to re-install the dependencies, QEMU or the Guest kernel, but just test code changes made to the runtime and
|
||||
agent. This can be done by running `. ~/ccv0.sh -d rebuild_and_install_kata`. (*Note this does a hard checkout*
|
||||
agent. This can be done by running `~/ccv0.sh rebuild_and_install_kata`. (*Note this does a hard checkout*
|
||||
*from git, so if your changes are only made locally it is better to do the individual steps e.g.*
|
||||
`. ~/ccv0.sh -d build_kata_runtime && . ~/ccv0.sh -d build_and_add_agent_to_rootfs && . ~/ccv0.sh -d build_and_install_rootfs`).
|
||||
`~/ccv0.sh build_kata_runtime && ~/ccv0.sh build_and_add_agent_to_rootfs && ~/ccv0.sh build_and_install_rootfs`).
|
||||
There are commands for a lot of steps in building, setting up and testing and the full list can be seen by running
|
||||
`~/ccv0.sh help`:
|
||||
```
|
||||
|
||||
Loading…
Reference in New Issue
Block a user