From f19811df434459ce60a9cf789cb187d1caab497a Mon Sep 17 00:00:00 2001 From: stevenhorsman Date: Fri, 21 Jan 2022 18:33:30 +0000 Subject: [PATCH 1/2] doc: Improve ccv0.sh and CCv0 how-to document General doc enchancements including: - Change `cd`s for `pushd` and `popd`s - Remove hard coded architectures - Tighten up the security where we `chmod 777` - Add support for not running as source - Updates so it doesn't do `ctr pull` if the image is on the local system already - Doc and Test running as non-root user (covered by #2879) - Update doc to match image_rpc changes Fixes: #3549 Fixes: #2879 Signed-off-by: stevenhorsman --- docs/how-to/ccv0.sh | 165 ++++++++++++++-------- docs/how-to/how-to-build-and-test-ccv0.md | 102 +++++++------ 2 files changed, 166 insertions(+), 101 deletions(-) diff --git a/docs/how-to/ccv0.sh b/docs/how-to/ccv0.sh index 2de97a183d..cb4ca847a6 100755 --- a/docs/how-to/ccv0.sh +++ b/docs/how-to/ccv0.sh @@ -1,6 +1,6 @@ #!/bin/bash -e # -# Copyright (c) 2021 IBM Corporation +# Copyright (c) 2021, 2022 IBM Corporation # # SPDX-License-Identifier: Apache-2.0 # @@ -8,7 +8,8 @@ # Disclaimer: This script is work in progress for supporting the CCv0 prototype # It shouldn't be considered supported by the Kata Containers community, or anyone else -# Based on https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md, but with elements of the tests/.ci scripts used +# Based on https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md, +# but with elements of the tests/.ci scripts used readonly script_name="$(basename "${BASH_SOURCE[0]}")" @@ -31,8 +32,14 @@ export tests_repo="${tests_repo:-github.com/kata-containers/tests}" export tests_branch="${tests_branch:-CCv0}" export target_branch=${tests_branch} # kata-containers/ci/lib.sh uses target branch var to check out tests repo +# if .bash_profile exists then use it, otherwise fall back to .profile +export PROFILE="${HOME}/.profile" +if [ -r "${HOME}/.bash_profile" ]; then + export PROFILE="${HOME}/.bash_profile" +fi + # Create a bunch of common, derived values up front so we don't need to create them in all the different functions -. "$HOME/.profile" +. ${PROFILE} if [ -z ${GOPATH} ]; then export GOPATH=${HOME}/go fi @@ -41,6 +48,8 @@ export katacontainers_repo_dir="${GOPATH}/src/${katacontainers_repo}" export ROOTFS_DIR="${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder/rootfs" export PULL_IMAGE="${PULL_IMAGE:-quay.io/kata-containers/confidential-containers:signed}" # Doesn't need authentication export CONTAINER_ID="${CONTAINER_ID:-0123456789}" +source /etc/os-release || source /usr/lib/os-release +grep -Eq "\" /etc/os-release 2> /dev/null && export USE_PODMAN=true debug_output() { if [ -n "${DEBUG}" ] @@ -65,7 +74,7 @@ usage() { Overview: Build and test kata containers from source Optionally set kata-containers and tests repo and branch as exported variables before running - e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && . ~/${script_name} -d build_and_install_all + e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/${script_name} build_and_install_all Usage: ${script_name} [options] Commands: @@ -129,26 +138,30 @@ rebuild_and_install_kata() { # Based on the jenkins_job_build.sh script in kata-containers/tests/.ci - checks out source code and installs dependencies initialize() { - # We need git to checkout and bootstrap the ci scripts - sudo apt-get update && sudo apt-get install -y git socat qemu-utils + # We need git to checkout and bootstrap the ci scripts and some other packages used in testing + sudo apt-get update && sudo apt-get install -y curl git socat qemu-utils PROFILE="${HOME}/.profile" grep -qxF "export GOPATH=\${HOME}/go" "${PROFILE}" || echo "export GOPATH=\${HOME}/go" >> "${PROFILE}" grep -qxF "export GOROOT=/usr/local/go" "${PROFILE}" || echo "export GOROOT=/usr/local/go" >> "${PROFILE}" - grep -qxF "export PATH=\${GOPATH}/bin:/usr/local/go/bin:/usr/sbin:/sbin:\${PATH}" "${PROFILE}" || echo "export PATH=\${GOPATH}/bin:/usr/local/go/bin:/usr/sbin:/sbin:\${PATH}" >> "${PROFILE}" - . "${HOME}/.profile" + grep -qxF "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" "${PROFILE}" || echo "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" >> "${PROFILE}" + + # Load the new go and PATH parameters from the profile + . ${PROFILE} mkdir -p "${GOPATH}" check_out_repos pushd "${tests_repo_dir}" ci_dir_name=".ci" - "${ci_dir_name}/install_go.sh" -p -f - "${ci_dir_name}/install_rust.sh" + sudo -E PATH=$PATH -s "${ci_dir_name}/install_go.sh" -p -f + sudo -E PATH=$PATH -s "${ci_dir_name}/install_rust.sh" # Run setup, but don't install kata as we will build it ourselves in locations matching the developer guide export INSTALL_KATA="no" - ${ci_dir_name}/setup.sh + sudo -E PATH=$PATH -s ${ci_dir_name}/setup.sh + # Reload the profile to pick up installed dependencies + . ${PROFILE} popd } @@ -177,11 +190,12 @@ check_out_repos() { } build_and_install_kata_runtime() { - cd ${katacontainers_repo_dir}/src/runtime + pushd ${katacontainers_repo_dir}/src/runtime make clean && make && sudo -E PATH=$PATH make install debug_output "We should have created Kata runtime binaries:: /usr/local/bin/kata-runtime and /usr/local/bin/containerd-shim-kata-v2" debug_output "We should have made the Kata configuration file: /usr/share/defaults/kata-containers/configuration.toml" debug_output "kata-runtime version: $(kata-runtime version)" + popd } configure() { @@ -195,11 +209,11 @@ configure() { # insert the cri_handler = "cc" into the [plugins.cri.containerd.runtimes.kata] section sudo sed -z -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' /etc/containerd/config.toml fi - + # Add cni directory to containerd config echo " [plugins.cri.cni] # conf_dir is the directory in which the admin places a CNI conf. - conf_dir = \"/etc/cni/net.d\"" >> /etc/containerd/config.toml + conf_dir = \"/etc/cni/net.d\"" | sudo tee -a /etc/containerd/config.toml # Switch image offload to true in kata config sudo sed -i -e 's/^# *\(service_offload\).*=.*$/\1 = true/g' /etc/kata-containers/configuration.toml @@ -232,21 +246,23 @@ build_and_add_agent_to_rootfs() { } build_a_custom_kata_agent() { - if [[ ! -L /bin/musl-g++ ]] - then - rustup target add x86_64-unknown-linux-musl - sudo ln -s /usr/bin/g++ /bin/musl-g++ - fi . "$HOME/.cargo/env" - cd ${katacontainers_repo_dir}/src/agent && make - debug_output "Kata agent built: $(ls -al ${katacontainers_repo_dir}/src/agent/target/x86_64-unknown-linux-musl/release/kata-agent)" + pushd ${katacontainers_repo_dir}/src/agent + sudo -E PATH=$PATH make + + ARCH=$(uname -m) + [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl + [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le + + debug_output "Kata agent built: $(ls -al ${katacontainers_repo_dir}/src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent)" # Run a make install into the rootfs directory in order to create the kata-agent.service file which is required when we add to the rootfs sudo -E PATH=$PATH make install DESTDIR="${ROOTFS}" + popd } create_a_local_rootfs() { sudo rm -rf "${ROOTFS_DIR}" - cd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder + pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder export distro="ubuntu" [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}" sudo -E OS_VERSION="${OS_VERSION:-}" GOPATH=$GOPATH DEBUG="${DEBUG}" USE_DOCKER="${use_docker:-}" SKOPEO=${SKOPEO:-} UMOCI=yes SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro} @@ -255,17 +271,23 @@ create_a_local_rootfs() { pushd "${tests_repo_dir}" git checkout ${tests_branch} popd - # During the ./rootfs.sh call the kata agent is built as root, so we need to update the permissions, so we can rebuild it sudo chown -R ${USER}:${USER} "${katacontainers_repo_dir}/src/agent/" + popd } add_custom_agent_to_rootfs() { - cd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder - sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent + pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder + + ARCH=$(uname -m) + [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl + [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le + + sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ${katacontainers_repo_dir}/src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/ sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/ debug_output "Added kata agent to rootfs: $(ls -al ${ROOTFS_DIR}/usr/bin/kata-agent)" + popd } build_and_install_rootfs() { @@ -274,12 +296,15 @@ build_and_install_rootfs() { } build_rootfs_image() { - cd ${katacontainers_repo_dir}/tools/osbuilder/image-builder - script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}' + pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder + # Logic from install_kata_image.sh - if we aren't using podman (ie on a fedora like), then use docker + [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}" + sudo -E USE_DOCKER="${use_docker:-}" ./image_builder.sh ${ROOTFS_DIR} + popd } install_rootfs_image() { - cd ${katacontainers_repo_dir}/tools/osbuilder/image-builder + pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder commit=$(git log --format=%h -1 HEAD) date=$(date +%Y-%m-%d-%T.%N%z) image="kata-containers-${date}-${commit}" @@ -287,15 +312,17 @@ install_rootfs_image() { (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img) echo "Built Rootfs from ${ROOTFS_DIR} to /usr/share/kata-containers/${image}" ls -al /usr/share/kata-containers/ + popd } install_guest_kernel_image() { - cd ${katacontainers_repo_dir}/tools/packaging/kernel - ./build-kernel.sh setup - ./build-kernel.sh build - sudo chmod 777 /usr/share/kata-containers/ # Give user permission to install kernel - ./build-kernel.sh install + pushd ${katacontainers_repo_dir}/tools/packaging/kernel + sudo -E PATH=$PATH ./build-kernel.sh setup + sudo -E PATH=$PATH ./build-kernel.sh build + sudo chmod u+wrx /usr/share/kata-containers/ # Give user permission to install kernel + sudo -E PATH=$PATH ./build-kernel.sh install debug_output "New kernel installed to $(ls -al /usr/share/kata-containers/vmlinux*)" + popd } build_qemu() { @@ -307,17 +334,29 @@ check_kata_runtime() { } init_kubernetes() { - # If kubernetes init has previous run we need to clean it by removing the image and resetting k8s - cid=$(docker ps -a -q -f name=^/kata-registry$) - if [ -n "${cid}" ]; then - docker stop ${cid} && docker rm ${cid} - fi - k8s_nodes=$(kubectl get nodes -o name) - if [ -n "${k8s_nodes}" ]; then - kubeadm reset -f + + # Check that kubeadm was installed and install it otherwise + if ! [ -x "$(command -v kubeadm)" ]; then + pushd "${tests_repo_dir}/.ci" + sudo -E PATH=$PATH -s install_kubernetes.sh + if [ "${CRI_CONTAINERD}" == "yes" ]; then + sudo -E PATH=$PATH -s "configure_containerd_for_kubernetes.sh" + fi + popd fi - export CI="true" && ${tests_repo_dir}/integration/kubernetes/init.sh + # If kubernetes init has previously run we need to clean it by removing the image and resetting k8s + cid=$(sudo docker ps -a -q -f name=^/kata-registry$) + if [ -n "${cid}" ]; then + sudo docker stop ${cid} && sudo docker rm ${cid} + fi + k8s_nodes=$(kubectl get nodes -o name 2>/dev/null || true) + if [ -n "${k8s_nodes}" ]; then + sudo kubeadm reset -f + fi + + export CI="true" && sudo -E PATH=$PATH -s ${tests_repo_dir}/integration/kubernetes/init.sh + sudo chown ${USER}:$(id -g -n ${USER}) "$HOME/.kube/config" cat << EOT | tee ~/nginx-kata.yaml apiVersion: v1 kind: Pod @@ -343,8 +382,8 @@ metadata: EOF # If already exists then delete and re-create - if [ -n "$(crictl pods --name ${crictl_sandbox_name} -q)" ]; then - crictl_delete_cc_pod + if [ -n "$(sudo crictl pods --name ${crictl_sandbox_name} -q)" ]; then + crictl_delete_cc fi pod_id=$(sudo crictl runp -r kata ~/pod-config.yaml) @@ -363,16 +402,18 @@ command: log_path: kata-cc.0.log EOF - pod_id=$(crictl pods --name ${crictl_sandbox_name} -q) + pod_id=$(sudo crictl pods --name ${crictl_sandbox_name} -q) container_id=$(sudo crictl create -with-pull ${pod_id} ~/container-config.yaml ~/pod-config.yaml) sudo crictl start ${container_id} sudo crictl ps -a } crictl_delete_cc() { - pod_id=$(crictl pods --name ${crictl_sandbox_name} -q) - container_id=$(crictl ps --pod ${pod_id} -q) - sudo crictl stop ${container_id} && sudo crictl rm ${container_id} + pod_id=$(sudo crictl pods --name ${crictl_sandbox_name} -q) + container_id=$(sudo crictl ps --pod ${pod_id} -q) + if [ -n "${container_id}" ]; then + sudo crictl stop ${container_id} && sudo crictl rm ${container_id} + fi sudo crictl stopp ${pod_id} && sudo crictl rmp ${pod_id} } @@ -393,8 +434,11 @@ restart_kata_pod() { test_kata_runtime() { echo "Running ctr with the kata runtime..." test_image="docker.io/library/busybox:latest" - sudo ctr image pull "${test_image}" - # If you hit too many requests run `sudo ctr image pull "docker.io/library/busybox:latest" -u ` command and retry + if [ -z $(ctr images ls -q name=="${test_image}") ]; then + sudo ctr image pull "${test_image}" + # If you hit too many requests run + # `sudo ctr image pull "docker.io/library/busybox:latest" -u ` command and retry + fi sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "${test_image}" test-kata uname -a } @@ -411,7 +455,8 @@ run_kata_and_capture_logs() { } get_ids() { - guest_cid=$(ps -ef | grep qemu-system-x86_64 | egrep -o "guest-cid=[0-9]*" | cut -d= -f2) && sandbox_id=$(ps -ef | grep qemu | egrep -o "sandbox-[^,][^,]*" | sed 's/sandbox-//g' | awk '{print $1}') + guest_cid=$(sudo ss -H --vsock | awk '{print $6}' | cut -d: -f1) + sandbox_id=$(ps -ef | grep qemu | egrep -o "sandbox-[^,][^,]*" | sed 's/sandbox-//g' | awk '{print $1}') } open_kata_console() { @@ -421,7 +466,7 @@ open_kata_console() { open_kata_shell() { get_ids - sudo kata-runtime exec ${sandbox_id} + sudo -E "PATH=$PATH" kata-runtime exec ${sandbox_id} } build_bundle_dir_if_necessary() { @@ -441,8 +486,11 @@ build_agent_ctl() { if [ -e "${HOME}/.cargo/registry" ]; then sudo chown -R ${USER}:${USER} "${HOME}/.cargo/registry" fi - make - cd "./target/x86_64-unknown-linux-musl/release" + sudo -E PATH=$PATH -s make + ARCH=$(uname -m) + [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl + [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le + cd "./target/${ARCH}-unknown-linux-${LIBC}/release/" } run_agent_ctl_command() { @@ -466,7 +514,7 @@ agent_create_container() { shim_pull_image() { get_ids - ctr_shim_command="ctr --namespace k8s.io shim --id ${sandbox_id} pull-image ${PULL_IMAGE} ${CONTAINER_ID}" + ctr_shim_command="sudo ctr --namespace k8s.io shim --id ${sandbox_id} pull-image ${PULL_IMAGE} ${CONTAINER_ID}" echo "Issuing command '${ctr_shim_command}'" ${ctr_shim_command} } @@ -476,6 +524,7 @@ main() { case "$opt" in d) DEBUG="-d" + set -x ;; h) usage 0 @@ -564,12 +613,12 @@ main() { agent_pull_image) agent_pull_image ;; + shim_pull_image) + shim_pull_image + ;; agent_create_container) agent_create_container ;; - shim_pull_image) - shim_pull_image - ;; *) usage 1 ;; diff --git a/docs/how-to/how-to-build-and-test-ccv0.md b/docs/how-to/how-to-build-and-test-ccv0.md index 3d21a7f7c1..c80c780f41 100644 --- a/docs/how-to/how-to-build-and-test-ccv0.md +++ b/docs/how-to/how-to-build-and-test-ccv0.md @@ -18,8 +18,6 @@ we have working. ### Basic script set-up and optional environment variables In order to build, configure and demo the CCv0 functionality, these are the set-up steps I take: -> **Note**: I've only tested this script running as root. I hope to fix this under - [issue #2879](https://github.com/kata-containers/kata-containers/issues/2879) - Provision a new VM - *I choose a Ubuntu 20.04 8GB VM for this as I had one available. There are some dependences on apt-get installed* *packages, so these will need re-working to be compatible with other platforms.* @@ -45,15 +43,16 @@ $ chmod u+x ccv0.sh $ export SKOPEO=yes ``` `skopeo` is - required for verifying container image signatures of pulled images. + required for passing source credentials and verifying container image signatures using the kata agent. ### Using `crictl` to do end-to-end testing of provisioning a container with the unencrypted image pulled on the guest - Run the full build process with Kubernetes off, so it's configure doesn't interfere with `crictl` using: ```bash $ export KUBERNETES="no" - $ . ~/ccv0.sh -d build_and_install_all + $ ~/ccv0.sh -d build_and_install_all ``` + > **Note**: Much of this script has to be run as `sudo`, so you are likely to get prompted for your password. - *I run this script sourced just so that the required installed components are accessible on the `PATH` to the rest* *of the process without having to reload the session.* - The steps that `build_and_install_all` takes is: @@ -67,19 +66,27 @@ $ chmod u+x ccv0.sh 20.04 and has extra packages like `umoci` added. - Build the Kata guest kernel - Install QEMU + > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker + during matching `ERROR: toomanyrequests: Too Many Requests`. To get past + this, login into Docker Hub and pull the images used with: + > ```bash + > $ docker login + > $ sudo docker pull ubuntu + > ``` + > then re-run the command. - The first time this runs it may take a while, but subsequent runs will be quicker as more things are already installed and they can be further cut down by not running all the above steps [see "Additional script usage" below](#additional-script-usage) - Create a new kata sandbox pod using `crictl` with: ```bash - $ . ~/ccv0.sh crictl_create_cc_pod + $ ~/ccv0.sh crictl_create_cc_pod ``` - This creates a pod configuration file, creates the pod from this using `sudo crictl runp -r kata ~/pod-config.yaml` and runs `sudo crictl pods` to show the pod - Create a new kata confidential container with: ```bash - $ . ~/ccv0.sh crictl_create_cc_container + $ ~/ccv0.sh crictl_create_cc_container ``` - This creates a container (based on `busybox:1.33.1`) in the kata cc sandbox and prints a list of containers. This will have been created based on an image pulled in the kata pod sandbox/guest, not on the host machine. @@ -114,19 +121,16 @@ there. - To verify that the image wasn't pulled on the host system we can look at the shared sandbox on the host and we should only see a single bundle for the pause container as the `busybox` based container image should have been pulled on the guest: - - Change to the pod's shared directory with: + - Find all the `rootfs` directories under in the pod's shared directory with: ```bash - $ cd /run/kata-containers/shared/sandboxes/${pod_id}/shared - ``` - - Find all the `rootfs` directories under this with: - ```bash - $ find . -name rootfs + $ pod_id=$(ps -ef | grep qemu | egrep -o "sandbox-[^,][^,]*" | sed 's/sandbox-//g' | awk '{print $1}') + $ sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs ./e89596e9de45ef2a154a5164554c9816293ab757cfd7a53d593fa144192a9964/rootfs ``` which should only show a single `rootfs` directory if the container image was pulled on the guest, not the host - Looking that `rootfs` directory with ```bash - $ ls -ltr ./e89596e9de45ef2a154a5164554c9816293ab757cfd7a53d593fa144192a9964/rootfs/ + $ sudo ls -ltr $(sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs) ``` prints something similar to ``` @@ -155,11 +159,18 @@ but Kubernetes can be used to create a non-confidential kata pod using `ccv0.sh` as a single node cluster: ```bash $ export KUBERNETES="yes" - $ . ~/ccv0.sh -d build_and_install_all + $ ~/ccv0.sh build_and_install_all ``` - > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker - during "Store custom stress image in registry" matching `ERROR: toomanyrequests: Too Many Requests`. To get past - this, log into docker hub with `docker login` and re-run the step with `. ~/ccv0.sh -d init_kubernetes`. + > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker + during matching `ERROR: toomanyrequests: Too Many Requests`. To get past + this, login into Docker Hub and pull the images used with: + > ```bash + > $ docker login + > $ sudo docker pull registry:2 + > $ sudo docker pull nginx + > $ sudo docker pull ubuntu + > ``` + > then re-run the command. - Check that your Kubernetes cluster has been correctly set-up: ``` $ kubectl get nodes @@ -168,7 +179,7 @@ stevenh-ccv0-demo1.fyre.ibm.com Ready control-plane,master 3m33s v1.21. ``` - Create a kata pod: ``` -$ ~/ccv0.sh -d create_kata_pod +$ ~/ccv0.sh create_kata_pod pod/nginx-kata created NAME READY STATUS RESTARTS AGE nginx-kata 0/1 ContainerCreating 0 5s @@ -196,7 +207,7 @@ that aren't broadly supported end-to-end. Here are some examples: - For debugging purposed you can optionally create a new terminal on the VM and connect to the kata guest's console log: ```bash - $ ~/ccv0.sh -d open_kata_console + $ ~/ccv0.sh open_kata_console ``` - In the first terminal run the pull image on guest command against the Kata agent, via the shim (`containerd-shim-kata-v2`). This can be achieved using the [containerd](https://github.com/containerd/containerd) CLI tool, `ctr`, which can be used to @@ -215,7 +226,7 @@ the `ccv0.sh` script to automatically fill in the variables: below steps to run the pull image command against the agent directly. - Run the pull image agent endpoint with: ```bash - $ ~/ccv0.sh -d shim_pull_image + $ ~/ccv0.sh shim_pull_image ``` which we print the `ctr shim` command for reference - Alternatively you can issue the command directly to the kata-agent pull image endpoint, which also supports @@ -230,9 +241,12 @@ the `ccv0.sh` script to automatically fill in the variables: proof of concept to allow more images to be pulled and tested. Once we have support for getting keys into the kata guest using the attestation-agent and/or KBS I'd expect container registry credentials to be looked up using that mechanism. + + > **Note**: the native rust implementation doesn't current flow credentials at the moment, so use + the `skopeo` based implementation if they are needed now. - Run the pull image agent endpoint with ```bash - $ ~/ccv0.sh -d agent_pull_image + $ ~/ccv0.sh agent_pull_image ``` and you should see output which includes `Command PullImage (1 of 1) returned (Ok(()), false)` to indicate that the `PullImage` request was successful e.g. @@ -244,11 +258,11 @@ the `ccv0.sh` script to automatically fill in the variables: {"msg":"response received","level":"INFO","ts":"2021-09-15T08:40:43.828200633-07:00","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","pid":"830920","response":""} {"msg":"Command PullImage (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-15T08:40:43.828261708-07:00","subsystem":"rpc","pid":"830920","source":"kata-agent-ctl","version":"0.1.0","name":"kata-agent-ctl"} ``` - > **Note**: The first time that `~/ccv0.sh -d agent_pull_image` is run, the `agent-ctl` tool with be complied + > **Note**: The first time that `~/ccv0.sh agent_pull_image` is run, the `agent-ctl` tool will be built which may take a few minutes. - To validate that the image pull was successful, you can open a shell into the kata pod with: ```bash - $ ~/ccv0.sh -d open_kata_shell + $ ~/ccv0.sh open_kata_shell ``` - Check the `/run/kata-containers/` directory to verify that the container image bundle has been created in a directory named either `01234556789` (for the container id), or the container image name, e.g. @@ -264,9 +278,9 @@ the `ccv0.sh` script to automatically fill in the variables: -rw-r--r-- 1 root root 63584 Jan 20 16:45 sha256_be9faa75035c20288cde7d2cdeb6cd1f5f4dbcd845d3f86f7feab61c4eff9eb5.mtree ``` - Leave the kata shell by running: - ```bash - $ exit - ``` + ```bash + $ exit + ``` ## Verifying signed images @@ -306,14 +320,14 @@ image by running: ```bash $ export CONTAINER_ID="unprotected-unsigned" $ export PULL_IMAGE="docker.io/library/busybox:latest" - $ ~/ccv0.sh -d agent_pull_image + $ ~/ccv0.sh agent_pull_image ``` - This finishes with a return `Ok()` - To test that an unsigned image from our *protected* test container registry is rejected we can run: ```bash $ export CONTAINER_ID="protected-unsigned" $ export PULL_IMAGE="quay.io/kata-containers/confidential-containers:unsigned" - $ ~/ccv0.sh -d agent_pull_image + $ ~/ccv0.sh agent_pull_image ``` - This results in an `ERROR: API failed` message from `agent_ctl` and the Kata sandbox console log shows the correct cause that the signature we has was not valid for the unsigned image: @@ -324,7 +338,7 @@ image by running: ```bash $ export CONTAINER_ID="protected-signed" $ export PULL_IMAGE="quay.io/kata-containers/confidential-containers:signed" - $ ~/ccv0.sh -d agent_pull_image + $ ~/ccv0.sh agent_pull_image ``` - This finishes with a return `Ok()` - Finally to check the image with a valid signature, but invalid GPG key (the real trusted piece of information we really @@ -332,7 +346,7 @@ want to protect with the attestation agent in future) fails we can run: ```bash $ export CONTAINER_ID="protected-wrong-key" $ export PULL_IMAGE="quay.io/kata-containers/confidential-containers:other_signed" - $ ~/ccv0.sh -d agent_pull_image + $ ~/ccv0.sh agent_pull_image ``` - Again this results in an `ERROR: API failed` message from `agent_ctl` and the Kata sandbox console log shows a slightly different error: @@ -342,29 +356,31 @@ want to protect with the attestation agent in future) fails we can run: ``` - To confirm that the first and third tests create the image bundles correct we can open a shell into the kata pod with: ```bash - $ ~/ccv0.sh -d open_kata_shell + $ ~/ccv0.sh open_kata_shell ``` - In the pod we can check the directories the images bundles were unpacked to: ```bash - $ ls /run/kata-containers/unprotected-unsigned/ + $ ls -ltr /run/kata-containers/unprotected-unsigned/ ``` should show something like ``` - config.json - rootfs - sha256_824b88c5c38e2b931cfd471061a576a2ac1c165ef7adeae6662687031b9f9e07.mtree - umoci.json + total 72 + drwxr-xr-x 10 root root 200 Jan 1 1970 rootfs + -rw-r--r-- 1 root root 2977 Jan 26 16:06 config.json + -rw-r--r-- 1 root root 372 Jan 26 16:06 umoci.json + -rw-r--r-- 1 root root 63724 Jan 26 16:06 sha256_1612e16ff3f6b0d09eefdc4e9d5c5c0624f63032743e016585b095b958778016.mtree ``` and ```bash - $ ls /run/kata-containers/protected-signed/ + $ ls -ltr /run/kata-containers/protected-signed/ ``` should show something like ``` - config.json - rootfs - sha256_ebf391d3f0ba36d4b64999ebbeadc878d229faec8839254a1c2264cf47735841.mtree - umoci.json + total 72 + drwxr-xr-x 10 root root 200 Jan 1 1970 rootfs + -rw-r--r-- 1 root root 2977 Jan 26 16:07 config.json + -rw-r--r-- 1 root root 372 Jan 26 16:07 umoci.json + -rw-r--r-- 1 root root 63568 Jan 26 16:07 sha256_ebf391d3f0ba36d4b64999ebbeadc878d229faec8839254a1c2264cf47735841.mtree ``` ## Additional script usage @@ -372,9 +388,9 @@ want to protect with the attestation agent in future) fails we can run: As well as being able to use the script as above to build all of `kata-containers` from scratch it can be used to just re-build bits of it by running the script with different parameters. For example after the first build you will often not need to re-install the dependencies, QEMU or the Guest kernel, but just test code changes made to the runtime and -agent. This can be done by running `. ~/ccv0.sh -d rebuild_and_install_kata`. (*Note this does a hard checkout* +agent. This can be done by running `~/ccv0.sh rebuild_and_install_kata`. (*Note this does a hard checkout* *from git, so if your changes are only made locally it is better to do the individual steps e.g.* -`. ~/ccv0.sh -d build_kata_runtime && . ~/ccv0.sh -d build_and_add_agent_to_rootfs && . ~/ccv0.sh -d build_and_install_rootfs`). +`~/ccv0.sh build_kata_runtime && ~/ccv0.sh build_and_add_agent_to_rootfs && ~/ccv0.sh build_and_install_rootfs`). There are commands for a lot of steps in building, setting up and testing and the full list can be seen by running `~/ccv0.sh help`: ``` From f68f73a849dd1896742c721553e4328a6d26a51d Mon Sep 17 00:00:00 2001 From: stevenhorsman Date: Mon, 31 Jan 2022 18:31:15 +0000 Subject: [PATCH 2/2] CCv0: Add network debug packages Add extra packages for debugging purposes Signed-off-by: stevenhorsman --- docs/how-to/ccv0.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/ccv0.sh b/docs/how-to/ccv0.sh index cb4ca847a6..112083618d 100755 --- a/docs/how-to/ccv0.sh +++ b/docs/how-to/ccv0.sh @@ -265,7 +265,7 @@ create_a_local_rootfs() { pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder export distro="ubuntu" [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}" - sudo -E OS_VERSION="${OS_VERSION:-}" GOPATH=$GOPATH DEBUG="${DEBUG}" USE_DOCKER="${use_docker:-}" SKOPEO=${SKOPEO:-} UMOCI=yes SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro} + sudo -E OS_VERSION="${OS_VERSION:-}" GOPATH=$GOPATH EXTRA_PKGS="vim iputils-ping net-tools" DEBUG="${DEBUG}" USE_DOCKER="${use_docker:-}" SKOPEO=${SKOPEO:-} UMOCI=yes SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro} # Install_rust.sh during rootfs.sh switches us to the main branch of the tests repo, so switch back now pushd "${tests_repo_dir}"