diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md index 32dd3260db..ac28d85eb1 100644 --- a/docs/Developer-Guide.md +++ b/docs/Developer-Guide.md @@ -1,55 +1,54 @@ -* [Warning](#warning) -* [Assumptions](#assumptions) -* [Initial setup](#initial-setup) -* [Requirements to build individual components](#requirements-to-build-individual-components) -* [Build and install the Kata Containers runtime](#build-and-install-the-kata-containers-runtime) -* [Check hardware requirements](#check-hardware-requirements) - * [Configure to use initrd or rootfs image](#configure-to-use-initrd-or-rootfs-image) - * [Enable full debug](#enable-full-debug) - * [debug logs and shimv2](#debug-logs-and-shimv2) - * [Enabling full `containerd` debug](#enabling-full-containerd-debug) - * [Enabling just `containerd shim` debug](#enabling-just-containerd-shim-debug) - * [Enabling `CRI-O` and `shimv2` debug](#enabling-cri-o-and-shimv2-debug) - * [journald rate limiting](#journald-rate-limiting) - * [`systemd-journald` suppressing messages](#systemd-journald-suppressing-messages) - * [Disabling `systemd-journald` rate limiting](#disabling-systemd-journald-rate-limiting) -* [Create and install rootfs and initrd image](#create-and-install-rootfs-and-initrd-image) - * [Build a custom Kata agent - OPTIONAL](#build-a-custom-kata-agent---optional) - * [Get the osbuilder](#get-the-osbuilder) - * [Create a rootfs image](#create-a-rootfs-image) - * [Create a local rootfs](#create-a-local-rootfs) - * [Add a custom agent to the image - OPTIONAL](#add-a-custom-agent-to-the-image---optional) - * [Build a rootfs image](#build-a-rootfs-image) - * [Install the rootfs image](#install-the-rootfs-image) - * [Create an initrd image - OPTIONAL](#create-an-initrd-image---optional) - * [Create a local rootfs for initrd image](#create-a-local-rootfs-for-initrd-image) - * [Build an initrd image](#build-an-initrd-image) - * [Install the initrd image](#install-the-initrd-image) -* [Install guest kernel images](#install-guest-kernel-images) -* [Install a hypervisor](#install-a-hypervisor) - * [Build a custom QEMU](#build-a-custom-qemu) - * [Build a custom QEMU for aarch64/arm64 - REQUIRED](#build-a-custom-qemu-for-aarch64arm64---required) -* [Run Kata Containers with Containerd](#run-kata-containers-with-containerd) -* [Run Kata Containers with Kubernetes](#run-kata-containers-with-kubernetes) -* [Troubleshoot Kata Containers](#troubleshoot-kata-containers) -* [Appendices](#appendices) - * [Checking Docker default runtime](#checking-docker-default-runtime) - * [Set up a debug console](#set-up-a-debug-console) - * [Simple debug console setup](#simple-debug-console-setup) - * [Enable agent debug console](#enable-agent-debug-console) - * [Start `kata-monitor`](#start-kata-monitor) - * [Connect to debug console](#connect-to-debug-console) - * [Traditional debug console setup](#traditional-debug-console-setup) - * [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell) - * [Build the debug image](#build-the-debug-image) - * [Configure runtime for custom debug image](#configure-runtime-for-custom-debug-image) - * [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console) - * [Enabling debug console for QEMU](#enabling-debug-console-for-qemu) - * [Enabling debug console for cloud-hypervisor / firecracker](#enabling-debug-console-for-cloud-hypervisor--firecracker) - * [Create a container](#create-a-container) - * [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console) - * [Obtain details of the image](#obtain-details-of-the-image) - * [Capturing kernel boot logs](#capturing-kernel-boot-logs) +- [Warning](#warning) +- [Assumptions](#assumptions) +- [Initial setup](#initial-setup) +- [Requirements to build individual components](#requirements-to-build-individual-components) +- [Build and install the Kata Containers runtime](#build-and-install-the-kata-containers-runtime) +- [Check hardware requirements](#check-hardware-requirements) + - [Configure to use initrd or rootfs image](#configure-to-use-initrd-or-rootfs-image) + - [Enable full debug](#enable-full-debug) + - [debug logs and shimv2](#debug-logs-and-shimv2) + - [Enabling full `containerd` debug](#enabling-full-containerd-debug) + - [Enabling just `containerd shim` debug](#enabling-just-containerd-shim-debug) + - [Enabling `CRI-O` and `shimv2` debug](#enabling-cri-o-and-shimv2-debug) + - [journald rate limiting](#journald-rate-limiting) + - [`systemd-journald` suppressing messages](#systemd-journald-suppressing-messages) + - [Disabling `systemd-journald` rate limiting](#disabling-systemd-journald-rate-limiting) +- [Create and install rootfs and initrd image](#create-and-install-rootfs-and-initrd-image) + - [Build a custom Kata agent - OPTIONAL](#build-a-custom-kata-agent---optional) + - [Get the osbuilder](#get-the-osbuilder) + - [Create a rootfs image](#create-a-rootfs-image) + - [Create a local rootfs](#create-a-local-rootfs) + - [Add a custom agent to the image - OPTIONAL](#add-a-custom-agent-to-the-image---optional) + - [Build a rootfs image](#build-a-rootfs-image) + - [Install the rootfs image](#install-the-rootfs-image) + - [Create an initrd image - OPTIONAL](#create-an-initrd-image---optional) + - [Create a local rootfs for initrd image](#create-a-local-rootfs-for-initrd-image) + - [Build an initrd image](#build-an-initrd-image) + - [Install the initrd image](#install-the-initrd-image) +- [Install guest kernel images](#install-guest-kernel-images) +- [Install a hypervisor](#install-a-hypervisor) + - [Build a custom QEMU](#build-a-custom-qemu) + - [Build a custom QEMU for aarch64/arm64 - REQUIRED](#build-a-custom-qemu-for-aarch64arm64---required) +- [Run Kata Containers with Containerd](#run-kata-containers-with-containerd) +- [Run Kata Containers with Kubernetes](#run-kata-containers-with-kubernetes) +- [Troubleshoot Kata Containers](#troubleshoot-kata-containers) +- [Appendices](#appendices) + - [Checking Docker default runtime](#checking-docker-default-runtime) + - [Set up a debug console](#set-up-a-debug-console) + - [Simple debug console setup](#simple-debug-console-setup) + - [Enable agent debug console](#enable-agent-debug-console) + - [Connect to debug console](#connect-to-debug-console) + - [Traditional debug console setup](#traditional-debug-console-setup) + - [Create a custom image containing a shell](#create-a-custom-image-containing-a-shell) + - [Build the debug image](#build-the-debug-image) + - [Configure runtime for custom debug image](#configure-runtime-for-custom-debug-image) + - [Create a container](#create-a-container) + - [Connect to the virtual machine using the debug console](#connect-to-the-virtual-machine-using-the-debug-console) + - [Enabling debug console for QEMU](#enabling-debug-console-for-qemu) + - [Enabling debug console for cloud-hypervisor / firecracker](#enabling-debug-console-for-cloud-hypervisor--firecracker) + - [Connecting to the debug console](#connecting-to-the-debug-console) + - [Obtain details of the image](#obtain-details-of-the-image) + - [Capturing kernel boot logs](#capturing-kernel-boot-logs) # Warning @@ -382,22 +381,19 @@ You can build and install the guest kernel image as shown [here](../tools/packag # Install a hypervisor -When setting up Kata using a [packaged installation method](install/README.md#installing-on-a-linux-system), the `qemu-lite` hypervisor is installed automatically. For other installation methods, you will need to manually install a suitable hypervisor. +When setting up Kata using a [packaged installation method](install/README.md#installing-on-a-linux-system), the +`QEMU` VMM is installed automatically. Cloud-Hypervisor and Firecracker VMMs are available from the [release tarballs](https://github.com/kata-containers/kata-containers/releases), as well as through [`kata-deploy`](../tools/packaging/kata-deploy/README.md). +You may choose to manually build your VMM/hypervisor. ## Build a custom QEMU -Your QEMU directory need to be prepared with source code. Alternatively, you can use the [Kata containers QEMU](https://github.com/kata-containers/qemu/tree/master) and checkout the recommended branch: +Kata Containers makes use of upstream QEMU branch. The exact version +and repository utilized can be found by looking at the [versions file](../versions.yaml). -``` -$ go get -d github.com/kata-containers/qemu -$ qemu_branch=$(grep qemu-lite- ${GOPATH}/src/github.com/kata-containers/kata-containers/versions.yaml | cut -d '"' -f2) -$ cd ${GOPATH}/src/github.com/kata-containers/qemu -$ git checkout -b $qemu_branch remotes/origin/$qemu_branch -$ your_qemu_directory=${GOPATH}/src/github.com/kata-containers/qemu -``` - -To build a version of QEMU using the same options as the default `qemu-lite` version , you could use the `configure-hypervisor.sh` script: +Kata often utilizes patches for not-yet-upstream fixes for components, +including QEMU. These can be found in the [packaging/QEMU directory](../tools/packaging/qemu/patches) +To build utilizing the same options as Kata, you should make use of the `configure-hypervisor.sh` script. For example: ``` $ go get -d github.com/kata-containers/kata-containers/tools/packaging $ cd $your_qemu_directory @@ -407,6 +403,8 @@ $ make -j $(nproc) $ sudo -E make install ``` +See the [static-build script for QEMU](../tools/packaging/static-build/qemu/build-static-qemu.sh) for a reference on how to get, setup, configure and build QEMU for Kata. + ### Build a custom QEMU for aarch64/arm64 - REQUIRED > **Note:** > @@ -618,8 +616,11 @@ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_cons > **Note** Ports 1024 and 1025 are reserved for communication with the agent > and gathering of agent logs respectively. -Next, connect to the debug console. The VSOCKS paths vary slightly between -cloud-hypervisor and firecracker. +##### Connecting to the debug console + +Next, connect to the debug console. The VSOCKS paths vary slightly between each +VMM solution. + In case of cloud-hypervisor, connect to the `vsock` as shown: ``` $ sudo su -c 'cd /var/run/vc/vm/{sandbox_id}/root/ && socat stdin unix-connect:clh.sock' @@ -636,6 +637,12 @@ CONNECT 1026 **Note**: You need to press the `RETURN` key to see the shell prompt. + +For QEMU, connect to the `vsock` as shown: +``` +$ sudo su -c 'cd /var/run/vc/vm/{sandbox_id} && socat "stdin,raw,echo=0,escape=0x11" "unix-connect:console.sock" +``` + To disconnect from the virtual machine, type `CONTROL+q` (hold down the `CONTROL` key and press `q`). diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs index 492aad33d2..e099086fc6 100644 --- a/src/agent/rustjail/src/cgroups/fs/mod.rs +++ b/src/agent/rustjail/src/cgroups/fs/mod.rs @@ -1008,11 +1008,11 @@ impl Manager { }) } - pub fn update_cpuset_path(&self, cpuset_cpus: &str) -> Result<()> { - if cpuset_cpus == "" { + pub fn update_cpuset_path(&self, guest_cpuset: &str, container_cpuset: &str) -> Result<()> { + if guest_cpuset == "" { return Ok(()); } - info!(sl!(), "update_cpuset_path to: {}", cpuset_cpus); + info!(sl!(), "update_cpuset_path to: {}", guest_cpuset); let h = cgroups::hierarchies::auto(); let h = Box::new(&*h); @@ -1026,8 +1026,8 @@ impl Manager { let h = cgroups::hierarchies::auto(); let h = Box::new(&*h); let cg = load_or_create(h, &self.cpath); - let cpuset_controller: &CpuSetController = cg.controller_of().unwrap(); - let path = cpuset_controller.path(); + let container_cpuset_controller: &CpuSetController = cg.controller_of().unwrap(); + let path = container_cpuset_controller.path(); let container_path = Path::new(path); info!(sl!(), "container cpuset path: {:?}", &path); @@ -1036,11 +1036,9 @@ impl Manager { if ancestor == root_path { break; } - if ancestor != container_path { - paths.push(ancestor); - } + paths.push(ancestor); } - info!(sl!(), "paths to update cpuset: {:?}", &paths); + info!(sl!(), "parent paths to update cpuset: {:?}", &paths); let mut i = paths.len(); loop { @@ -1056,10 +1054,20 @@ impl Manager { .to_str() .unwrap() .trim_start_matches(root_path.to_str().unwrap()); - info!(sl!(), "updating cpuset for path {:?}", &r_path); + info!(sl!(), "updating cpuset for parent path {:?}", &r_path); let cg = load_or_create(h, &r_path); let cpuset_controller: &CpuSetController = cg.controller_of().unwrap(); - cpuset_controller.set_cpus(cpuset_cpus)?; + cpuset_controller.set_cpus(guest_cpuset)?; + } + + if !container_cpuset.is_empty() { + info!( + sl!(), + "updating cpuset for container path: {:?} cpuset: {}", + &container_path, + container_cpuset + ); + container_cpuset_controller.set_cpus(container_cpuset)?; } Ok(()) diff --git a/src/agent/rustjail/src/cgroups/mock.rs b/src/agent/rustjail/src/cgroups/mock.rs new file mode 100644 index 0000000000..e1603c8468 --- /dev/null +++ b/src/agent/rustjail/src/cgroups/mock.rs @@ -0,0 +1,74 @@ +// Copyright (c) 2020 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +use protobuf::{CachedSize, SingularPtrField, UnknownFields}; + +use crate::cgroups::Manager as CgroupManager; +use crate::protocols::agent::{BlkioStats, CgroupStats, CpuStats, MemoryStats, PidsStats}; +use anyhow::Result; +use cgroups::freezer::FreezerState; +use libc::{self, pid_t}; +use oci::LinuxResources; +use std::collections::HashMap; +use std::string::String; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Manager { + pub paths: HashMap, + pub mounts: HashMap, + pub cpath: String, +} + +impl CgroupManager for Manager { + fn apply(&self, _: pid_t) -> Result<()> { + Ok(()) + } + + fn set(&self, _: &LinuxResources, _: bool) -> Result<()> { + Ok(()) + } + + fn get_stats(&self) -> Result { + Ok(CgroupStats { + cpu_stats: SingularPtrField::some(CpuStats::default()), + memory_stats: SingularPtrField::some(MemoryStats::new()), + pids_stats: SingularPtrField::some(PidsStats::new()), + blkio_stats: SingularPtrField::some(BlkioStats::new()), + hugetlb_stats: HashMap::new(), + unknown_fields: UnknownFields::default(), + cached_size: CachedSize::default(), + }) + } + + fn freeze(&self, _: FreezerState) -> Result<()> { + Ok(()) + } + + fn destroy(&mut self) -> Result<()> { + Ok(()) + } + + fn get_pids(&self) -> Result> { + Ok(Vec::new()) + } +} + +impl Manager { + pub fn new(cpath: &str) -> Result { + Ok(Self { + paths: HashMap::new(), + mounts: HashMap::new(), + cpath: cpath.to_string(), + }) + } + + pub fn update_cpuset_path(&self, _: &str, _: &str) -> Result<()> { + Ok(()) + } + + pub fn get_cg_path(&self, _: &str) -> Option { + Some("".to_string()) + } +} diff --git a/src/agent/src/sandbox.rs b/src/agent/src/sandbox.rs index 5ba25218bc..53bf053dfd 100644 --- a/src/agent/src/sandbox.rs +++ b/src/agent/src/sandbox.rs @@ -236,14 +236,29 @@ impl Sandbox { return Ok(()); } - let cpuset = rustjail_cgroups::fs::get_guest_cpuset()?; + let guest_cpuset = rustjail_cgroups::fs::get_guest_cpuset()?; for (_, ctr) in self.containers.iter() { + let cpu = ctr + .config + .spec + .as_ref() + .unwrap() + .linux + .as_ref() + .unwrap() + .resources + .as_ref() + .unwrap() + .cpu + .as_ref(); + let container_cpust = if let Some(c) = cpu { &c.cpus } else { "" }; + info!(self.logger, "updating {}", ctr.id.as_str()); ctr.cgroup_manager .as_ref() .unwrap() - .update_cpuset_path(cpuset.as_str())?; + .update_cpuset_path(guest_cpuset.as_str(), &container_cpust)?; } Ok(()) diff --git a/tools/packaging/kata-deploy/k8s-1.18/kata-runtimeClasses.yaml b/tools/packaging/kata-deploy/k8s-1.18/kata-runtimeClasses.yaml new file mode 100644 index 0000000000..a696e3b3fe --- /dev/null +++ b/tools/packaging/kata-deploy/k8s-1.18/kata-runtimeClasses.yaml @@ -0,0 +1,40 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1beta1 +metadata: + name: kata-qemu-virtiofs +handler: kata-qemu-virtiofs +overhead: + podFixed: + memory: "160Mi" + cpu: "250m" +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1beta1 +metadata: + name: kata-qemu +handler: kata-qemu +overhead: + podFixed: + memory: "160Mi" + cpu: "250m" +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1beta1 +metadata: + name: kata-clh +handler: kata-clh +overhead: + podFixed: + memory: "130Mi" + cpu: "250m" +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1beta1 +metadata: + name: kata-fc +handler: kata-fc +overhead: + podFixed: + memory: "130Mi" + cpu: "250m" diff --git a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml index 5e80c0f890..bcda760b9f 100644 --- a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml +++ b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml @@ -18,7 +18,7 @@ spec: katacontainers.io/kata-runtime: cleanup containers: - name: kube-kata-cleanup - image: katadocker/kata-deploy + image: katadocker/kata-deploy:2.0.2 imagePullPolicy: Always command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh reset" ] env: diff --git a/tools/packaging/release/update-repository-version.sh b/tools/packaging/release/update-repository-version.sh index 2b51a8afe9..b8329aeb7e 100755 --- a/tools/packaging/release/update-repository-version.sh +++ b/tools/packaging/release/update-repository-version.sh @@ -110,6 +110,16 @@ bump_repo() { fi fi + if [ "${repo}" == "kata-containers" ]; then + info "Updating kata-deploy / kata-cleanup image tags" + sed -i "s#katadocker/kata-deploy:${current_version}#katadocker/kata-deploy:${new_version}#g" tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml + sed -i "s#katadocker/kata-deploy:${current_version}#katadocker/kata-deploy:${new_version}#g" tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml + git diff + + git add tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml + git add tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml + fi + info "Creating PR message" notes_file=notes.md cat <"${notes_file}"