From 9d4cd77560c4a7dfa71988b9b5b40d7c9493e29a Mon Sep 17 00:00:00 2001 From: stevenhorsman Date: Fri, 28 Jan 2022 14:29:01 +0000 Subject: [PATCH] doc: Add k8s e2e doc and scripts - Add scripts and documentation to build, configure and test created a Kata CC unencrypted container using Kubernetes - Switch test images to quay.io as image_rpc.rs has some problems with docker.io? - Update documentation to better fit the kata documentation requirements and fix typos - Fixes: #3511 Signed-off-by: stevenhorsman --- docs/how-to/ccv0.sh | 78 ++++------ docs/how-to/how-to-build-and-test-ccv0.md | 177 ++++++++++++---------- 2 files changed, 125 insertions(+), 130 deletions(-) diff --git a/docs/how-to/ccv0.sh b/docs/how-to/ccv0.sh index 112083618d..a31fe164de 100755 --- a/docs/how-to/ccv0.sh +++ b/docs/how-to/ccv0.sh @@ -94,9 +94,8 @@ Commands: - crictl_create_cc_pod Use crictl to create a new kata cc pod - crictl_create_cc_container Use crictl to create a new busybox container in the kata cc pod - crictl_delete_cc Use crictl to delete the kata cc pod sandbox and container in it -- create_kata_pod: Create a kata runtime nginx pod in Kubernetes -- delete_kata_pod: Delete a kata runtime nginx pod in Kubernetes -- restart_kata_pod: Delete the kata nginx pod, then re-create it +- kubernetes_create_cc_pod: Create a Kata CC runtime busybox-based pod in Kubernetes +- kubernetes_delete_cc_pod: Delete the Kata CC runtime busybox-based pod in Kubernetes - open_kata_console: Stream the kata runtime's console - open_kata_shell: Open a shell into the kata runtime - agent_pull_image: Run PullImage command against the agent with agent-ctl @@ -121,7 +120,6 @@ build_and_install_all() { build_and_install_rootfs install_guest_kernel_image build_qemu - build_bundle_dir_if_necessary check_kata_runtime if [ "${KUBERNETES}" == "yes" ]; then init_kubernetes @@ -204,11 +202,8 @@ configure() { # Temp PoC verify code: Inject policy path config parameter sudo sed -i -e 's%^kernel_params = "\(.*\)"%kernel_params = "\1 agent.container_policy_file=/etc/containers/quay_verification/quay_policy.json"%g' /etc/kata-containers/configuration.toml - # K8s doesn't fully work with kata cc in this enviornment yet issues #3511 - if [ "${KUBERNETES}" != "yes" ]; then - # insert the cri_handler = "cc" into the [plugins.cri.containerd.runtimes.kata] section - sudo sed -z -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' /etc/containerd/config.toml - fi + # insert the cri_handler = "cc" into the [plugins.cri.containerd.runtimes.kata] section + sudo sed -z -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' /etc/containerd/config.toml # Add cni directory to containerd config echo " [plugins.cri.cni] @@ -333,8 +328,8 @@ check_kata_runtime() { sudo kata-runtime check } +k8s_pod_file="${HOME}/busybox-cc.yaml" init_kubernetes() { - # Check that kubeadm was installed and install it otherwise if ! [ -x "$(command -v kubeadm)" ]; then pushd "${tests_repo_dir}/.ci" @@ -357,17 +352,27 @@ init_kubernetes() { export CI="true" && sudo -E PATH=$PATH -s ${tests_repo_dir}/integration/kubernetes/init.sh sudo chown ${USER}:$(id -g -n ${USER}) "$HOME/.kube/config" - cat << EOT | tee ~/nginx-kata.yaml + cat << EOF > ${k8s_pod_file} apiVersion: v1 kind: Pod metadata: - name: nginx-kata + name: busybox-cc spec: runtimeClassName: kata containers: - name: nginx - image: nginx -EOT + image: quay.io/kata-containers/confidential-containers:signed + imagePullPolicy: Always +EOF +} + +kubernetes_create_cc_pod() { + kubectl apply -f ${k8s_pod_file} + kubectl get pods +} + +kubernetes_delete_cc_pod() { + kubectl delete -f ${k8s_pod_file} } crictl_sandbox_name=kata-cc-busybox-sandbox @@ -376,7 +381,7 @@ crictl_create_cc_pod() { sudo iptables -P FORWARD ACCEPT # Create crictl pod config -cat << EOF > ~/pod-config.yaml +cat << EOF > ${HOME}/pod-config.yaml metadata: name: ${crictl_sandbox_name} EOF @@ -386,13 +391,13 @@ EOF crictl_delete_cc fi - pod_id=$(sudo crictl runp -r kata ~/pod-config.yaml) + pod_id=$(sudo crictl runp -r kata ${HOME}/pod-config.yaml) sudo crictl pods } crictl_create_cc_container() { # Create container configuration yaml based on our test copy of busybox - cat << EOF > ~/container-config.yaml + cat << EOF > ${HOME}/container-config.yaml metadata: name: kata-cc-busybox image: @@ -403,7 +408,7 @@ log_path: kata-cc.0.log EOF pod_id=$(sudo crictl pods --name ${crictl_sandbox_name} -q) - container_id=$(sudo crictl create -with-pull ${pod_id} ~/container-config.yaml ~/pod-config.yaml) + container_id=$(sudo crictl create -with-pull ${pod_id} ${HOME}/container-config.yaml ${HOME}/pod-config.yaml) sudo crictl start ${container_id} sudo crictl ps -a } @@ -417,27 +422,11 @@ crictl_delete_cc() { sudo crictl stopp ${pod_id} && sudo crictl rmp ${pod_id} } -create_kata_pod() { - kubectl apply -f ~/nginx-kata.yaml - kubectl get pods -} - -delete_kata_pod() { - kubectl delete -f ~/nginx-kata.yaml -} - -restart_kata_pod() { - delete_kata_pod - create_kata_pod -} - test_kata_runtime() { echo "Running ctr with the kata runtime..." - test_image="docker.io/library/busybox:latest" + test_image="quay.io/kata-containers/confidential-containers:signed" if [ -z $(ctr images ls -q name=="${test_image}") ]; then sudo ctr image pull "${test_image}" - # If you hit too many requests run - # `sudo ctr image pull "docker.io/library/busybox:latest" -u ` command and retry fi sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "${test_image}" test-kata uname -a } @@ -449,9 +438,9 @@ run_kata_and_capture_logs() { sudo systemctl start systemd-journald test_kata_runtime echo "Collecting logs..." - sudo journalctl -q -o cat -a -t kata-runtime > ~/kata-runtime.log - sudo journalctl -q -o cat -a -t kata > ~/shimv2.log - echo "Logs output to ~/kata-runtime.log and ~/shimv2.log" + sudo journalctl -q -o cat -a -t kata-runtime > ${HOME}/kata-runtime.log + sudo journalctl -q -o cat -a -t kata > ${HOME}/shimv2.log + echo "Logs output to ${HOME}/kata-runtime.log and ${HOME}/shimv2.log" } get_ids() { @@ -473,7 +462,7 @@ build_bundle_dir_if_necessary() { bundle_dir="/tmp/bundle" if [ ! -d "${bundle_dir}" ]; then rootfs_dir="$bundle_dir/rootfs" - image="busybox" + image="quay.io/kata-containers/confidential-containers:signed" mkdir -p "$rootfs_dir" && (cd "$bundle_dir" && runc spec) sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf - fi @@ -589,14 +578,11 @@ main() { crictl_delete_cc) crictl_delete_cc ;; - create_kata_pod) - create_kata_pod + kubernetes_create_cc_pod) + kubernetes_create_cc_pod ;; - delete_kata_pod) - delete_kata_pod - ;; - restart_kata_pod) - restart_kata_pod + kubernetes_delete_cc_pod) + kubernetes_delete_cc_pod ;; test) test_kata_runtime diff --git a/docs/how-to/how-to-build-and-test-ccv0.md b/docs/how-to/how-to-build-and-test-ccv0.md index c80c780f41..7ae9d0eec1 100644 --- a/docs/how-to/how-to-build-and-test-ccv0.md +++ b/docs/how-to/how-to-build-and-test-ccv0.md @@ -37,15 +37,20 @@ $ chmod u+x ccv0.sh $ export tests_branch=stevenh/add-ccv0-changes-to-build ``` before running the script. - - By default `ccv0.sh` enables the agent to use the rust implementation to pull container images on the guest. If - you wish to instead build and include the `skopeo` package for this then run + - By default `ccv0.sh` enables the agent to use the rust implementation to pull container images on the guest. + If you wish to instead build and include the `skopeo` package for this then run: ```bash $ export SKOPEO=yes ``` `skopeo` is required for passing source credentials and verifying container image signatures using the kata agent. -### Using `crictl` to do end-to-end testing of provisioning a container with the unencrypted image pulled on the guest +- At this point you can provision a Kata confidential containers pod and container with either + [`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image), + or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image) + and then test and use it. + +### Using `crictl` for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image - Run the full build process with Kubernetes off, so it's configure doesn't interfere with `crictl` using: ```bash @@ -61,7 +66,7 @@ $ chmod u+x ccv0.sh - Use the `tests/.ci` scripts to install the build dependencies - Build and install the Kata runtime - Configure Kata to use containerd and for debug and confidential containers features to be enabled (including - enabling console access to the kata-runtime, which should only be done in development) + enabling console access to the Kata guest shell, which should only be done in development) - Create, build and install a rootfs for the Kata hypervisor to use. For 'CCv0' this is currently based on Ubuntu 20.04 and has extra packages like `umoci` added. - Build the Kata guest kernel @@ -70,7 +75,7 @@ $ chmod u+x ccv0.sh during matching `ERROR: toomanyrequests: Too Many Requests`. To get past this, login into Docker Hub and pull the images used with: > ```bash - > $ docker login + > $ sudo docker login > $ sudo docker pull ubuntu > ``` > then re-run the command. @@ -78,18 +83,78 @@ $ chmod u+x ccv0.sh installed and they can be further cut down by not running all the above steps [see "Additional script usage" below](#additional-script-usage) -- Create a new kata sandbox pod using `crictl` with: +- Create a new Kata sandbox pod using `crictl` with: ```bash $ ~/ccv0.sh crictl_create_cc_pod ``` - This creates a pod configuration file, creates the pod from this using `sudo crictl runp -r kata ~/pod-config.yaml` and runs `sudo crictl pods` to show the pod -- Create a new kata confidential container with: +- Create a new Kata confidential container with: ```bash $ ~/ccv0.sh crictl_create_cc_container ``` - - This creates a container (based on `busybox:1.33.1`) in the kata cc sandbox and prints a list of containers. - This will have been created based on an image pulled in the kata pod sandbox/guest, not on the host machine. + - This creates a container (based on `busybox:1.33.1`) in the Kata cc sandbox and prints a list of containers. + This will have been created based on an image pulled in the Kata pod sandbox/guest, not on the host machine. + +As this point you should have a `crictl` pod and container that is using the Kata confidential containers runtime. +You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest) +or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim) + +#### Clean up the `crictl` pod sandbox and container +- When the testing is complete you can delete the container and pod by running: + ```bash + $ ~/ccv0.sh crictl_delete_cc + ``` +### Using Kubernetes for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image + +- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is configured and created using the VM + as a single node cluster: + ```bash + $ export KUBERNETES="yes" + $ ~/ccv0.sh build_and_install_all + ``` + > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker + during matching `ERROR: toomanyrequests: Too Many Requests`. To get past + this, login into Docker Hub and pull the images used with: + > ```bash + > $ sudo docker login + > $ sudo docker pull registry:2 + > $ sudo docker pull ubuntu:20.04 + > ``` + > then re-run the command. +- Check that your Kubernetes cluster has been correctly set-up by running : + ```bash + $ kubectl get nodes + ``` + and checking that you see a single node e.g. + ```text + NAME STATUS ROLES AGE VERSION + stevenh-ccv0-k8s1.fyre.ibm.com Ready control-plane,master 43s v1.22.0 + ``` +- Create a Kata confidential containers pod by running: + ```bash + $ ~/ccv0.sh kubernetes_create_cc_pod + ``` +- Wait a few seconds for pod to start then check that the pod's status is `Running` with + ```bash + $ kubectl get pods + ``` + which should show something like: + ```text + NAME READY STATUS RESTARTS AGE + busybox-cc 1/1 Running 0 54s + ``` + +- As this point you should have a Kubernetes pod and container running, that is using the Kata +confidential containers runtime. +You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest) +or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim) + +#### Clean up the Kubernetes pod sandbox and container +- When the testing is complete you can delete the container and pod by running: + ```bash + $ ~/ccv0.sh kubernetes_delete_cc_pod + ``` ### Validate that the container image was pulled on the guest @@ -97,7 +162,7 @@ There are a couple of ways we can check that the container pull image action was the guest's file system for the unpacked bundle and checking the host's directories to ensure it wasn't also pulled there. - To check the guest's file system: - - Open a shell into the sandbox with: + - Open a shell into the Kata guest with: ```bash $ ~/ccv0.sh open_kata_shell ``` @@ -114,7 +179,7 @@ there. drwxr-xr-x 12 root root 240 Jan 20 10:03 rootfs ``` which shows how the image has been pulled and then unbundled on the guest. - - Leave the kata shell by running: + - Leave the Kata guest shell by running: ```bash $ exit ``` @@ -125,14 +190,13 @@ there. ```bash $ pod_id=$(ps -ef | grep qemu | egrep -o "sandbox-[^,][^,]*" | sed 's/sandbox-//g' | awk '{print $1}') $ sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs - ./e89596e9de45ef2a154a5164554c9816293ab757cfd7a53d593fa144192a9964/rootfs ``` which should only show a single `rootfs` directory if the container image was pulled on the guest, not the host - Looking that `rootfs` directory with ```bash $ sudo ls -ltr $(sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs) ``` - prints something similar to + shows something similar to ``` total 668 -rwxr-xr-x 1 root root 682696 Aug 25 13:58 pause @@ -141,71 +205,17 @@ there. drwxr-xr-x 2 root root 6 Jan 20 02:01 sys drwxr-xr-x 2 root root 25 Jan 20 02:01 etc ``` - which is clearly the pause container indicating that the `busybox` based container image if not exposed to the host. + which is clearly the pause container indicating that the `busybox` based container image is not exposed to the host. -#### Clean up `crictl` pod sandbox and container -- When the testing is complete you can either continue on with different tests (mentioned below) using the pod sandbox, or delete the container and pod by running: - ```bash - $ ~/ccv0.sh crictl_delete_cc - ``` - -### Setting up Kubernetes - -The documentation for end-to-end testing of a confidential container created through Kubernetes -[is not completed yet](https://github.com/kata-containers/kata-containers/issues/3511), -but Kubernetes can be used to create a non-confidential kata pod using `ccv0.sh`. - -- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is configured and created using the VM - as a single node cluster: - ```bash - $ export KUBERNETES="yes" - $ ~/ccv0.sh build_and_install_all - ``` - > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker - during matching `ERROR: toomanyrequests: Too Many Requests`. To get past - this, login into Docker Hub and pull the images used with: - > ```bash - > $ docker login - > $ sudo docker pull registry:2 - > $ sudo docker pull nginx - > $ sudo docker pull ubuntu - > ``` - > then re-run the command. -- Check that your Kubernetes cluster has been correctly set-up: -``` -$ kubectl get nodes -NAME STATUS ROLES AGE VERSION -stevenh-ccv0-demo1.fyre.ibm.com Ready control-plane,master 3m33s v1.21.1 -``` -- Create a kata pod: -``` -$ ~/ccv0.sh create_kata_pod -pod/nginx-kata created -NAME READY STATUS RESTARTS AGE -nginx-kata 0/1 ContainerCreating 0 5s -``` -- Wait a few seconds for pod to start -``` -$ kubectl get pods -NAME READY STATUS RESTARTS AGE -nginx-kata 1/1 Running 0 29s -``` -- This Kubernetes pod can now be used for further testing (mentioned below) using the created kata pod sandbox, or deleted - by running - ```bash - $ ~/ccv0.sh delete_kata_pod - ``` - - -### Using a kata pod sandbox for testing with `agent-ctl` or `ctr shim` +### Using a Kata pod sandbox for testing with `agent-ctl` or `ctr shim` Once you have a kata pod sandbox created as described above, either using -[`crictl`](#using-crictl-to-do-end-to-end-testing-of-provisioning-a-container-with-the-unencrypted-image-pulled-on-the-guest) -or [Kubernetes](#setting-up-kubernetes), you can use this to test specific components of the kata confidential +[`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image), or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image) +, you can use this to test specific components of the Kata confidential containers architecture. This can be useful for development and debugging to isolate and test features that aren't broadly supported end-to-end. Here are some examples: -- For debugging purposed you can optionally create a new terminal on the VM and connect to the kata guest's console log: +- For debugging purposed you can optionally create a new terminal on the VM and connect to the Kata guest's console log: ```bash $ ~/ccv0.sh open_kata_console ``` @@ -229,7 +239,7 @@ the `ccv0.sh` script to automatically fill in the variables: $ ~/ccv0.sh shim_pull_image ``` which we print the `ctr shim` command for reference -- Alternatively you can issue the command directly to the kata-agent pull image endpoint, which also supports +- Alternatively you can issue the command directly to the `kata-agent` pull image endpoint, which also supports credentials in order to pull from an authenticated registry: - Optionally set up some environment variables to set the image and credentials used: - Set the `PULL_IMAGE` environment variable e.g. `export PULL_IMAGE="docker.io/library/busybox:latest"` @@ -239,7 +249,7 @@ the `ccv0.sh` script to automatically fill in the variables: `export SOURCE_CREDS=":"` > **Note**: the credentials support on the agent request is a tactical solution for the short-term proof of concept to allow more images to be pulled and tested. Once we have support for getting - keys into the kata guest using the attestation-agent and/or KBS I'd expect container registry + keys into the Kata guest image using the attestation-agent and/or KBS I'd expect container registry credentials to be looked up using that mechanism. > **Note**: the native rust implementation doesn't current flow credentials at the moment, so use @@ -260,7 +270,7 @@ the `ccv0.sh` script to automatically fill in the variables: ``` > **Note**: The first time that `~/ccv0.sh agent_pull_image` is run, the `agent-ctl` tool will be built which may take a few minutes. -- To validate that the image pull was successful, you can open a shell into the kata pod with: +- To validate that the image pull was successful, you can open a shell into the Kata guest with: ```bash $ ~/ccv0.sh open_kata_shell ``` @@ -277,7 +287,7 @@ the `ccv0.sh` script to automatically fill in the variables: -rw-r--r-- 1 root root 372 Jan 20 16:45 umoci.json -rw-r--r-- 1 root root 63584 Jan 20 16:45 sha256_be9faa75035c20288cde7d2cdeb6cd1f5f4dbcd845d3f86f7feab61c4eff9eb5.mtree ``` -- Leave the kata shell by running: +- Leave the Kata shell by running: ```bash $ exit ``` @@ -312,9 +322,9 @@ to the `kernel_params` entry in `/etc/kata-containers/configuration.toml`. With this policy parameter set a few tests of image verification can be done to test different scenarios > **Note**: at the time of writing the `ctr shim` command has a [bug](https://github.com/kata-containers/kata-containers/issues/3020), so I'm using the agent commands directly through `agent-ctl` to drive the tests -- If you don't already have a kata pod sandbox created, follow the instructions above to create one either using - [`crictl`](#using-crictl-to-do-end-to-end-testing-of-provisioning-a-container-with-the-unencrypted-image-pulled-on-the-guest) - or [Kubernetes](#setting-up-kubernetes) +- If you don't already have a Kata pod sandbox created, follow the instructions above to create one either using + [`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image) + or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image) - To test the fallback behaviour works using an unsigned image on an *unprotected* registry we can pull the `busybox` image by running: ```bash @@ -354,7 +364,7 @@ want to protect with the attestation agent in future) fails we can run: FATA[0001] Source image rejected: Invalid GPG signature... ``` -- To confirm that the first and third tests create the image bundles correct we can open a shell into the kata pod with: +- To confirm that the first and third tests create the image bundles correct we can open a shell into the Kata pod with: ```bash $ ~/ccv0.sh open_kata_shell ``` @@ -398,7 +408,7 @@ $ ~/ccv0.sh help Overview: Build and test kata containers from source Optionally set kata-containers and tests repo and branch as exported variables before running - e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && . ~/ccv0.sh -d build_and_install_all + e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/ccv0.sh build_and_install_all Usage: ccv0.sh [options] Commands: @@ -418,9 +428,8 @@ Commands: - crictl_create_cc_pod Use crictl to create a new kata cc pod - crictl_create_cc_container Use crictl to create a new busybox container in the kata cc pod - crictl_delete_cc Use crictl to delete the kata cc pod sandbox and container in it -- create_kata_pod: Create a kata runtime nginx pod in Kubernetes -- delete_kata_pod: Delete a kata runtime nginx pod in Kubernetes -- restart_kata_pod: Delete the kata nginx pod, then re-create it +- kubernetes_create_cc_pod: Create a Kata CC runtime busybox-based pod in Kubernetes +- kubernetes_delete_cc_pod: Delete the Kata CC runtime busybox-based pod in Kubernetes - open_kata_console: Stream the kata runtime's console - open_kata_shell: Open a shell into the kata runtime - agent_pull_image: Run PullImage command against the agent with agent-ctl