From 608e003abcf7ce54c12512d79b5196b74c1883db Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 24 Feb 2022 12:41:33 +1100 Subject: [PATCH 01/26] agent: Don't attempt to create directories for hugepage configuration allocate_hugepages() constructs the path for the sysfs directory containing hugepage configuration, then attempts to create this directory if it does not exist. This doesn't make sense: sysfs is a view into kernel configuration, if the kernel has support for the hugepage size, the directory will already be there, if it doesn't, trying to create it won't help. For the same reason, attempting to create the "nr_hugepages" file itself is pointless, so there's no reason to call OpenOptions::create(true). Signed-off-by: David Gibson --- src/agent/src/mount.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index d7dbc08ef4..17ae85bda7 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -335,20 +335,16 @@ fn allocate_hugepages(logger: &Logger, options: &[String]) -> Result<()> { // sysfs entry is always of the form hugepages-${pagesize}kB // Ref: https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt - let path = Path::new(SYS_FS_HUGEPAGES_PREFIX).join(format!("hugepages-{}kB", pagesize / 1024)); - - if !path.exists() { - fs::create_dir_all(&path).context("create hugepages-size directory")?; - } + let path = Path::new(SYS_FS_HUGEPAGES_PREFIX) + .join(format!("hugepages-{}kB", pagesize / 1024)) + .join("nr_hugepages"); // write numpages to nr_hugepages file. - let path = path.join("nr_hugepages"); let numpages = format!("{}", size / pagesize); info!(logger, "write {} pages to {:?}", &numpages, &path); let mut file = OpenOptions::new() .write(true) - .create(true) .open(&path) .context(format!("open nr_hugepages directory {:?}", &path))?; From 42e35505b0fb1f207ca041a4d22de05d41106be8 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 24 Feb 2022 12:53:49 +1100 Subject: [PATCH 02/26] agent: Verify that we allocated as many hugepages as we need allocate_hugepages() writes to the kernel sysfs file to allocate hugepages in the Kata VM. However, even if the write succeeds, it's not certain that the kernel will actually be able to allocate as many hugepages as we requested. This patch reads back the file after writing it to check if we were able to allocate all the required hugepages. fixes #3816 Signed-off-by: David Gibson --- src/agent/src/mount.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index 17ae85bda7..41671e897f 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -351,6 +351,23 @@ fn allocate_hugepages(logger: &Logger, options: &[String]) -> Result<()> { file.write_all(numpages.as_bytes()) .context(format!("write nr_hugepages failed: {:?}", &path))?; + // Even if the write succeeds, the kernel isn't guaranteed to be + // able to allocate all the pages we requested. Verify that it + // did. + let verify = fs::read_to_string(&path).context(format!("reading {:?}", &path))?; + let allocated = verify + .trim_end() + .parse::() + .map_err(|_| anyhow!("Unexpected text {:?} in {:?}", &verify, &path))?; + if allocated != size / pagesize { + return Err(anyhow!( + "Only allocated {} of {} hugepages of size {}", + allocated, + numpages, + pagesize + )); + } + Ok(()) } From 4adf93ef2c7ebf75ad6c662a98726c286dd3f87e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 7 Mar 2022 19:08:56 +0100 Subject: [PATCH 03/26] tools: release: Do not consider release candidates as stable releases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit During the release of 2.4.0-rc0 @egernst noticed an incositency in the way we handle release tags, as release candidates are being taken as "stable" releases, while both the kata-deploy tests and the release action consider this as "latest". Ideally we should have our own tag for "release candidate", but that's something that could and should be discussed more extensively outside of the scope of this quick fix. For now, let's align the code generating the PR for bumping the release with what we already do as part of the release action and kata-deploy test, and tag "-rc" as latest, regardless of which branch it's coming from. Fixes: #3847 Signed-off-by: Fabiano Fidêncio --- .../release/update-repository-version.sh | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/tools/packaging/release/update-repository-version.sh b/tools/packaging/release/update-repository-version.sh index 088792de03..9dc1b3cc46 100755 --- a/tools/packaging/release/update-repository-version.sh +++ b/tools/packaging/release/update-repository-version.sh @@ -68,7 +68,6 @@ generate_kata_deploy_commit() { kata-deploy files must be adapted to a new release. The cases where it happens are when the release goes from -> to: * main -> stable: - * kata-deploy / kata-cleanup: change from \"latest\" to \"rc0\" * kata-deploy-stable / kata-cleanup-stable: are removed * stable -> stable: @@ -161,7 +160,7 @@ bump_repo() { # +----------------+----------------+ # | from | to | # -------------------+----------------+----------------+ - # kata-deploy | "latest" | "rc0" | + # kata-deploy | "latest" | "latest" | # -------------------+----------------+----------------+ # kata-deploy-stable | "stable" | REMOVED | # -------------------+----------------+----------------+ @@ -183,23 +182,32 @@ bump_repo() { info "Updating kata-deploy / kata-cleanup image tags" local version_to_replace="${current_version}" local replacement="${new_version}" + local removed_files=false if [ "${target_branch}" == "main" ]; then if [[ "${new_version}" =~ "rc" ]]; then ## this is the case 2) where we remove te kata-deploy / kata-cleanup stable files git rm "${kata_deploy_stable_yaml}" git rm "${kata_cleanup_stable_yaml}" - else - ## this is the case 1) where we just do nothing + removed_files=true + fi + + ## these are the cases 1) and 2), where "alpha" and "rc0" are tagged as "latest" + version_to_replace="latest" + replacement="latest" + else + if [[ "${new_version}" =~ "rc" ]]; then + ## we're in a stable branch, coming from "rcX" (tagged as latest) and we're going + ## to "rcX+1", which should still be tagged as latest. + version_to_replace="latest" replacement="latest" fi - version_to_replace="latest" fi - if [ "${version_to_replace}" != "${replacement}" ]; then - ## this covers case 2) and 3), as on both of them we have changes on kata-deploy / kata-cleanup files - sed -i "s#${registry}:${version_to_replace}#${registry}:${new_version}#g" "${kata_deploy_yaml}" - sed -i "s#${registry}:${version_to_replace}#${registry}:${new_version}#g" "${kata_cleanup_yaml}" + if [ "${version_to_replace}" != "${replacement}" ] || [ "${removed_files}" == "true" ]; then + ## this covers case 3), as it has changes on kata-deploy / kata-cleanup files + sed -i "s#${registry}:${version_to_replace}#${registry}:${replacement}#g" "${kata_deploy_yaml}" + sed -i "s#${registry}:${version_to_replace}#${registry}:${replacement}#g" "${kata_cleanup_yaml}" git diff From 84dff4405786cd98b727d97f9ed9228806cfbda9 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Mon, 7 Mar 2022 11:15:25 -0800 Subject: [PATCH 04/26] release: Adapt kata-deploy for 2.4.0-rc0 kata-deploy files must be adapted to a new release. The cases where it happens are when the release goes from -> to: * main -> stable: * kata-deploy-stable / kata-cleanup-stable: are removed * stable -> stable: * kata-deploy / kata-cleanup: bump the release to the new one. There are no changes when doing an alpha release, as the files on the "main" branch always point to the "latest" and "stable" tags. Signed-off-by: Eric Ernst --- .../base/kata-cleanup-stable.yaml | 46 ------------- .../kata-deploy/base/kata-deploy-stable.yaml | 69 ------------------- 2 files changed, 115 deletions(-) delete mode 100644 tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml delete mode 100644 tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml diff --git a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml deleted file mode 100644 index f1d9d0a2f9..0000000000 --- a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml +++ /dev/null @@ -1,46 +0,0 @@ ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kubelet-kata-cleanup - namespace: kube-system -spec: - selector: - matchLabels: - name: kubelet-kata-cleanup - template: - metadata: - labels: - name: kubelet-kata-cleanup - spec: - serviceAccountName: kata-label-node - nodeSelector: - katacontainers.io/kata-runtime: cleanup - containers: - - name: kube-kata-cleanup - image: quay.io/kata-containers/kata-deploy:stable - imagePullPolicy: Always - command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh reset" ] - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - privileged: false - volumeMounts: - - name: dbus - mountPath: /var/run/dbus - - name: systemd - mountPath: /run/systemd - volumes: - - name: dbus - hostPath: - path: /var/run/dbus - - name: systemd - hostPath: - path: /run/systemd - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml deleted file mode 100644 index 346e4c0ee2..0000000000 --- a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml +++ /dev/null @@ -1,69 +0,0 @@ ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kata-deploy - namespace: kube-system -spec: - selector: - matchLabels: - name: kata-deploy - template: - metadata: - labels: - name: kata-deploy - spec: - serviceAccountName: kata-label-node - containers: - - name: kube-kata - image: quay.io/kata-containers/kata-deploy:stable - imagePullPolicy: Always - lifecycle: - preStop: - exec: - command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"] - command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh install" ] - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - privileged: false - volumeMounts: - - name: crio-conf - mountPath: /etc/crio/ - - name: containerd-conf - mountPath: /etc/containerd/ - - name: kata-artifacts - mountPath: /opt/kata/ - - name: dbus - mountPath: /var/run/dbus - - name: systemd - mountPath: /run/systemd - - name: local-bin - mountPath: /usr/local/bin/ - volumes: - - name: crio-conf - hostPath: - path: /etc/crio/ - - name: containerd-conf - hostPath: - path: /etc/containerd/ - - name: kata-artifacts - hostPath: - path: /opt/kata/ - type: DirectoryOrCreate - - name: dbus - hostPath: - path: /var/run/dbus - - name: systemd - hostPath: - path: /run/systemd - - name: local-bin - hostPath: - path: /usr/local/bin/ - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate From a4dcaf3cf41b983349399c15670237d3ab178dc8 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Mon, 7 Mar 2022 11:15:25 -0800 Subject: [PATCH 05/26] release: Kata Containers 2.4.0-rc0 - Enhancement: fix comments/logs and delete not used function - storage: make k8s emptyDir volume creation location configurable - Implement direct-assigned volume - Bump containerd to 1.6.1 - experimentally enable vcpu hotplug and virtio-mem on arm64 in kernel part - versions: Upgrade to Cloud Hypervisor v22.0 - katatestutils: remove distro constraints - Minor fixes for the `disable_block_device_use` comments - clh: stop virtofsd if clh fails to boot up the vm - clh: tdx: Don't use sharedFS with Confidential Guests - runtime: Build golang components with extra security options - snap: Use git clone depth 1 for QEMU and dependencies - snap: Don't build cloud-hypevisor on ppc64le - build: always reset ARCH after getting it - virtcontainers: remove temp dir created for vsock in test code - docs: Add unit testing presentation - virtcontainers: Use available s390x hugepages - Update QEMU >= 6.1.0 in configure-hypervisor.sh - Fix monitor listen address - snap: clh: Re-use kata-deploy script here - osbuilder: Add CentOS Stream rootfs - runtime: Gofmt fixes - Update `confidential_guest` comments - cleanup runtime pkgs for Darwin build, add basic Darwin build/unit test - docs: Update Readme document - runtime: use Cmd.StdoutPipe instead of self-created pipe - docs: Developer-Guide build a custom Kata agent with musl - kata-agent: Fix mismatching error of cgroup and mountinfo. - runtime, config: make selinux configurable - Fix unbound variable / typo on error mesage - clh: Add TDX support - virtcontainers: Do not add a virtio-rng-ccw device - kata-monitor: fix collecting metrics for sandboxes not started through CRI - runtime: fix package declaration for ppc64le - Make the hypervisor framework not Linux specific - kata-deploy: Simplify Dockerfile and support s390x - Support nerdctl OCI hooks - shim: log events for CRI-O - docs: Update contributing link - kata-deploy: Use (kata with) qemu as the default shim-v2 binary - kata-monitor: simplify sandbox cache management and attach kubernetes POD metadata to metrics - nydus: add lazyload support for kata with clh - kernel: remove SYS_SUPPORTS_HUGETLBFS from powerpc fragments - packaging: Use `patch` for applying patches - virtcontainers: Remove duplicated assert messages in utils test code - versions: add nydus-snapshotter - docs: Update limitations document - packaging: support qemu-tdx - Kata manager fix install - versions: Linux 5.15.x - trace-forwarder/agent-ctl: run cargo fmt/clippy in make check - docs: Improve top-level README - runtime: use github.com/mdlayher/vsock@v1.1.0 - tools: Build cloud-hypervisor with "--features tdx" - virtiofsd: Use "-o announce_submounts" - feature: hugepages support - tools: clh: Allow to set when to build from sources and the build flags passed down to cargo - docs: Remove docker run and shared memory from limitations - versions: Udpate Cloud Hypervisor to 55479a64d237 - kernel: add missing config fragment for TDx - runtime: The index variable is initialized multiple times in for - scripts: fix a typo while to check build_type - versions: bump CRI-O to its 1.23 release - feature(nydusd): add nydusd support to introduce lazyload ability - docs: Fix relative links in Markdown - kernel: support TDx - device: Actually update PCIDEVICE_ environment variables for the guest - docs: Update link to EFK stack docs - runtime: support QEMU SGX - snap: update qemu version to 6.1.0 for arm - Release process related fixes - openshift-ci: switch to CentOS Stream - virtcontainers: Split the rootless package into OS specific parts - runtime: suppport split firmware - kata-deploy: for testing, make sure we use the PR branch - docs: Remove Zun documentation with kata containers - agent: Fix execute_hook() args error - workflows: stop checking revert commit 84dff440 release: Adapt kata-deploy for 2.4.0-rc0 b257e0e5 rustjail: delete function signal in BaseContainer d647b28b agent: delete meaningless FIXME comment 1b34494b runtime: fix invalid comments for pkg/resourcecontrol afc567a9 storage: make k8s emptyDir creation configurable e76519af runtime: small refactor to improve readability 7e5f11a5 vendor: Update containerd to 1.6.1 42771fa7 runtime: don't set socket and thread for arm/virt 8828ef41 kernel: add arm experimental kernel build support 8a9007fe config: remove 2 config as they are removed in 5.15 1b6f7401 kernel: add arm experimental patches to support vcpu hotplug and virtio-mem f905161b runtime: mount direct-assigned block device fs only once 27fb4902 agent: add get volume stats handler in agent ea51ef1c runtime: forward the stat and resize requests from shimv2 to kata agent c39281ad runtime: update container creation to work with direct assigned volumes 4e00c237 agent: add grpc interface for stat and resize operations e9b5a255 runtime: add stat and resize APIs to containerd-shim-v2 6e0090ab runtime: persist direct volume mount info fa326b4e runtime: augment kata-runtime CLI to support direct-assigned volume b8844fb8 versions: Upgrade to Cloud Hypervisor v22.0 af804734 clh: stop virtofsd if clh fails to boot up the vm 97951a2d clh: Don't use SharedFS with Confidential Guests c30b3a9f clh: Adding a volume is not supported without SharedFS f889f1f9 clh: introduce supportsSharedFS() 54d27ed7 clh: introduce loadVirtiofsDaemon() ae2221ea clh: introduce stopVirtiofsDaemon() e8bc26f9 clh: introduce setupVirtiofsDaemon() 413b3b47 clh: introduce createVirtiofsDaemon() 55cd0c89 runtime: Build golang components with extra security options 76e4f6a2 Revert "hypervisors: Confidential Guests do not support Device hotplug" fa8b9392 config: qemu: Fix disable_block_device_use comments 9615c8bc config: fc: Don't expose disable_block_device_use c1fb4bb7 snap: Don't build cloud-hypevisor on ppc64le 58913694 snap: Use git clone depth 1 for QEMU and dependencies b27c7f40 docs: Add unit testing presentation e64c54a2 monitor: Listen to localhost only by default e6350d3d monitor: Fix build options a67b93bb snap: clh: Re-use kata-deploy script here f31125fe version: Bump cloud-hypervisor to b0324f85571c441f 54d0a672 subsystem: build edf20766 docs: Update Readme document eda8ea15 runtime: Gofmt fixes 4afb278f ci: add github action to exercise darwin build, unit tests e355a718 container: file is not linux specific b31876ee device-manager: move linux-only test to a linux-only file 6a5c6344 resourcecontrol: SystemdCgroup check is not necessarily linux specific cc58cf69 resourcecontrol: convert stats dev_t to unit64types 5be188cc utils: Add darwin stub ad044919 virtcontainers: Convert stats dev_t to uint64 56751089 katautils: Use a syscall wrapper for the hook JSON state 7d64ae7a runtime: Add a syscall wrapper package abc681ca katautils: Add Darwin stub for the netNS API de574662 config: Expand confidential_guest comments 641d475f config: clh: Use "Intel TDX" instead of just "TDX" 0bafa2de config: clh: Mention supported TEEs 81ed269e runtime: use Cmd.StdoutPipe instead of self-created pipe 8edca8bb kata-agent: Fix mismatching error of cgroup and mountinfo. a9ba7c13 clh: Fix typo on HotplugRemoveDevice 827ab82a tools: clh: Fix unbound variable 082d538c runtime: make selinux configurable 1103f5a4 virtcontainers: Use FilesystemSharer for sharing the containers files 533c1c0e virtcontainers: Keep all filesystem sharing prep code to sandbox.go 61590bbd virtcontainers: Add a Linux implementation for the FilesystemSharer 03fc1cbd virtcontainers: Add a filesystem sharing interface 72434333 clh: Add TDX support a13b4d5a clh: Add firmware to the config file a8827e0c hypervisors: Confidential Guests do not support NVDIMM f50ff9f7 hypervisors: Confidential Guests do not support Memory hotplug df8ffecd hypervisors: Confidential Guests do not support Device hotplug 28c4c044 hypervisors: Confidential Guests do not support VCPUs hotplug 29ee870d clh: Add confidential_guest to the config file 9621c596 clh: refactor image / initrd configuration set dcdc412e clh: use common kernel params from the hypervisor code 4c164afb versions: Update Cloud Hypervisor to 5343e09e7b8db b2a65f90 virtcontainers: Use available s390x hugepages cb4230e6 runtime: fix package declaration for ppc64le fec26f8e kata-monitor: trivial: rename symbols & labels 9fd4e551 runtime: Move the resourcecontrol package one layer up 823faee8 virtcontainers: Rename the cgroups package 0d1a7da6 virtcontainers: Rename and clean the cgroup interface ad10e201 virtcontainers: cgroups: Move non Linux routine to utils.go d49d0b6f virtcontainers: cgroups: Define a cgroup interface 3ac52e81 kata-monitor: fix updating sandbox cache at startup 160bb621 kata-monitor: bump version to 0.3.0 1a3381b0 docs: Developer-Guide build a custom Kata agent with musl f6fc1621 shim: log events for CRI-O 1d68a08f docs: Update contributing link 9123fc09 kata-deploy: Simplify Dockerfile and support s390x 11220f05 kata-deploy: Use (kata with) qemu as the default shim-v2 binary 3175aad5 virtiofs-nydus: add lazyload support for kata with clh 94b831eb virtcontainers: remove temp dir created for vsock in test code 8cc1b186 kernel: remove SYS_SUPPORTS_HUGETLBFS from powerpc fragments 5c9d2b41 packaging: Use `patch` for applying patches 5b3fb6f8 kernel: Build SGX as part of the vanilla kernel 2c35d8cb workflows: Stop building the experimental kernel 32e7845d snap: Build vanilla kernel for all arches 27de212f runtime: Always add network endpoints from the pod netns 1cee0a94 virtcontainers: Remove duplicated assert messages in utils test code 6c1d149a docs: Update limitations document 7c4ee6ec packaging/qemu: create no_patches file for qemu-tdx d47c488b versions: add qemu tdx section 77c29bfd container: Remove VFIO lazy attach handling 7241d618 versions: add nydus-snapshotter 26b3f001 virtcontainers: Split hypervisor into Linux and OS agnostic bits fa0e9dc6 virtcontainers: Make all Linux VMMs only build on Linux c91035d0 virtcontainers: Move non QEMU specific constants to hypervisor.go 10ae0591 virtcontainers: Move guest protection definitions to hypervisor.go b28d0274 virtcontainers: Make max vCPU config less QEMU specific a5f6df6a govmm: Define the number of supported vCPUs per architecture a6b40151 tools: clh: Remove unused variables 5816c132 tools: Build cloud-hypervisor with "--features tdx" e6060cb7 versions: Linux 5.15.x 9818cf71 docs: Improve top-level and runtime README 36c3fc12 agent: support hugepages for containers 81a8baa5 runtime: add hugepages support 7df677c0 runtime: Update calculateSandboxMemory to include Hugepages Limit 948a2b09 tools: clh: Ensure the download binary is executable 72bf5496 agent: handle hook process result 80e8dbf1 agent: valid envs for hooks 4f96e3ea katautils: Pass the nerdctl netns annotation to the OCI hooks a871a33b katautils: Run the createRuntime hooks d9dfce14 katautils: Run the preStart hook in the host namespace 6be6d0a3 katautils: Pass the OCI annotations back to the called OCI hooks 493ebc8c utils: Update kata manager docs 34b2e67d utils: Added more kata manager cli options 714c9f56 utils: Improve containerd configuration c464f326 utils: kata-manager: Force containerd sym link creation 4755d004 utils: Fix unused parameter 601be4e6 utils: Fix containerd installation ae21fcc7 utils: Fix Kata tar archive check f4d1e45c utils: Add kata-manager CLI options for kata and containerd 395cff48 docs: Remove docker run and shared memory from limitations e07545a2 tools: clh: Allow passing down a build flag 55cdef22 tools: clh: Add the possibility to always build from sources 3f87835a utils: Switch kata manager to use getopts 4bd945b6 virtiofsd: Use "-o announce_submounts" 37df1678 build: always reset ARCH after getting it 3a641b56 katatestutils: remove distro constraints 90fd625d versions: Udpate Cloud Hypervisor to 55479a64d237 573a37b3 osbuilder: Add CentOS Stream rootfs f10642c8 osbuilder: Source .cargo/env before checking Rust 955d359f kernel: add missing config fragment for TDx 734b618c agent-ctl: run cargo fmt/clippy in make check 12c37faf trace-forwarder: add make check for Rust c1ce67d9 runtime: use github.com/mdlayher/vsock@v1.1.0 42a878e6 runtime: The index variable is initialized multiple times in for 1797b3eb packaging/kernel: build TDX guest kernel 98752529 versions: add url and tag for tdx kernel bc8464e0 packaging/kernel: add option -s option 2d9f89ae feature(nydusd): add nydusd support to introduse lazyload ability b19b6938 docs: Fix relative links in Markdown 9590874d device: Update PCIDEVICE_ environment variables for the guest 7b7f426a device: Keep host to VM PCI mapping persistently 0b2bd641 device: Rework update_spec_pci() to update_env_pci() 982f14fa runtime: support QEMU SGX 40aa43f4 docs: Update link to EFK stack docs 54e1faec scripts: fix a typo while to check build_type 07b9d93f virtcontainer: Simplify the sandbox network creation flow 2c7087ff virtcontainers: Make all endpoints Linux only 49d2cde1 virtcontainers: Split network tests into generic and OS specific parts 0269077e virtcontainers: Remove the netlink package dependency from network.go 7fca5792 virtcontainers: Unify Network endpoints management interface c67109a2 virtcontainers: Remove the Network PostAdd method e0b26443 virtcontainers: Define a Network interface 5e119e90 virtcontainers: Rename the Network structure fields and methods b858d0de virtcontainers: Make all Network fields private 49eee79f virtcontainers: Remove the NetworkNamespace structure 844eb619 virtcontainers: Have CreateVM use a Network reference d7b67a7d virtcontainers: Network API cleanups and simplifications 2edea883 virtcontainers: Make the Network structure manage endpoints 8f48e283 virtcontainers: Expand the Network structure 5ef522f7 runtime: check kvm module `sev` correctly 419d8134 snap: update qemu version to 6.1.0 for arm 00722187 docs: update Release-Process.md 496bc10d tools: check for yq before using it 88a70d32 Revert "workflows: Ensure a label change re-triggers the actions" a9bebb31 openshift-ci: switch to CentOS Stream 89047901 kata-deploy-push: only run if PR modifying tools path 7ffe9e51 virtcontainers: Do not add a virtio-rng-ccw device 1f29478b runtime: suppport split firmware 24796d2f kata-deploy: for testing, make sure we use the PR branch 1cc1c8d0 docs: Remove images from Zun documentation 5861e52f docs: Remove Zun documentation with kata containers 903a6a45 versions: Bump critools to its 1.23 release 63eb1158 versions: bump CRI-O to its 1.23 release 5083ae65 workflows: stop checking revert commit 14e7f52a virtcontainers: Split the rootless package into OS specific parts ab447285 kata-monitor: add kubernetes pod metadata labels to metrics 834e199e kata-monitor: drop unused functions 7516a8c5 kata-monitor: rework the sandbox cache sync with the container manager e78d80ea kata-monitor: silently ignore CHMOD events on the sandboxes fs e9eb34ce kata-monitor: improve debug logging 4fc4c76b agent: Fix execute_hook() args error Signed-off-by: Eric Ernst --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 171f1d5b9c..cbc70e35ba 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-alpha2 +2.4.0-rc0 From 2b41d275a603eaad6e2df3f3606695ce3e0e96d3 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Mon, 7 Mar 2022 11:15:29 -0800 Subject: [PATCH 06/26] release: Revert kata-deploy changes after 2.4.0-rc0 release As 2.4.0-rc0 has been released, let's switch the kata-deploy / kata-cleanup tags back to "latest", and re-add the kata-deploy-stable and the kata-cleanup-stable files. Signed-off-by: Eric Ernst --- .../base/kata-cleanup-stable.yaml | 46 +++++++++++++ .../kata-deploy/base/kata-deploy-stable.yaml | 69 +++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml create mode 100644 tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml diff --git a/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml new file mode 100644 index 0000000000..f1d9d0a2f9 --- /dev/null +++ b/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kubelet-kata-cleanup + namespace: kube-system +spec: + selector: + matchLabels: + name: kubelet-kata-cleanup + template: + metadata: + labels: + name: kubelet-kata-cleanup + spec: + serviceAccountName: kata-label-node + nodeSelector: + katacontainers.io/kata-runtime: cleanup + containers: + - name: kube-kata-cleanup + image: quay.io/kata-containers/kata-deploy:stable + imagePullPolicy: Always + command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh reset" ] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + privileged: false + volumeMounts: + - name: dbus + mountPath: /var/run/dbus + - name: systemd + mountPath: /run/systemd + volumes: + - name: dbus + hostPath: + path: /var/run/dbus + - name: systemd + hostPath: + path: /run/systemd + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate diff --git a/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml new file mode 100644 index 0000000000..346e4c0ee2 --- /dev/null +++ b/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy-stable.yaml @@ -0,0 +1,69 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kata-deploy + namespace: kube-system +spec: + selector: + matchLabels: + name: kata-deploy + template: + metadata: + labels: + name: kata-deploy + spec: + serviceAccountName: kata-label-node + containers: + - name: kube-kata + image: quay.io/kata-containers/kata-deploy:stable + imagePullPolicy: Always + lifecycle: + preStop: + exec: + command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"] + command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh install" ] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + privileged: false + volumeMounts: + - name: crio-conf + mountPath: /etc/crio/ + - name: containerd-conf + mountPath: /etc/containerd/ + - name: kata-artifacts + mountPath: /opt/kata/ + - name: dbus + mountPath: /var/run/dbus + - name: systemd + mountPath: /run/systemd + - name: local-bin + mountPath: /usr/local/bin/ + volumes: + - name: crio-conf + hostPath: + path: /etc/crio/ + - name: containerd-conf + hostPath: + path: /etc/containerd/ + - name: kata-artifacts + hostPath: + path: /opt/kata/ + type: DirectoryOrCreate + - name: dbus + hostPath: + path: /var/run/dbus + - name: systemd + hostPath: + path: /run/systemd + - name: local-bin + hostPath: + path: /usr/local/bin/ + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate From 6a850899c9294b425f6b4781c255ba6171eee561 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Mon, 7 Mar 2022 09:55:15 +0000 Subject: [PATCH 07/26] CI: Create GHA to add PR sizing label Created a new GitHub Action workflow file that adds a sizing label to each PR. Fixes: #3841. Signed-off-by: James O. D. Hunt --- .github/workflows/add-pr-sizing-label.yaml | 38 ++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/add-pr-sizing-label.yaml diff --git a/.github/workflows/add-pr-sizing-label.yaml b/.github/workflows/add-pr-sizing-label.yaml new file mode 100644 index 0000000000..751a942762 --- /dev/null +++ b/.github/workflows/add-pr-sizing-label.yaml @@ -0,0 +1,38 @@ +# Copyright (c) 2022 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +name: Add PR sizing label + +on: + pull_request_target: + types: + - opened + - reopened + - synchronize + +jobs: + add-pr-size-label: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v1 + + - name: Install PR sizing label script + run: | + # Clone into a temporary directory to avoid overwriting + # any existing github directory. + pushd $(mktemp -d) &>/dev/null + git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts + sudo install pr-add-size-label.sh /usr/local/bin + popd &>/dev/null + + - name: Add PR sizing label + env: + GITHUB_TOKEN: ${{ secrets.KATA_GITHUB_ACTIONS_TOKEN }} + run: | + pr=${{ github.event.number }} + sudo apt -y install diffstat patchutils + + pr-add-size-label.sh -p "$pr" From 5ec7592dfae71526a577d041b67f9630e638ddf5 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Tue, 8 Mar 2022 09:42:05 -0600 Subject: [PATCH 08/26] kernel: fix cve-2022-0847 bump guest kernel version to fix cve-2022-0847 "Dirty Pipe" fixes #3852 Signed-off-by: Julio Montes --- tools/packaging/kernel/kata_config_version | 2 +- versions.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version index 8643cf6deb..d61f00d8ca 100644 --- a/tools/packaging/kernel/kata_config_version +++ b/tools/packaging/kernel/kata_config_version @@ -1 +1 @@ -89 +90 diff --git a/versions.yaml b/versions.yaml index 2412053e5f..11badc6eb5 100644 --- a/versions.yaml +++ b/versions.yaml @@ -153,7 +153,7 @@ assets: kernel: description: "Linux kernel optimised for virtual machines" url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/" - version: "v5.15.23" + version: "v5.15.26" tdx: description: "Linux kernel that supports TDX" url: "https://github.com/intel/tdx/archive/refs/tags" From ffdf961ae99a7c19e0651c3c8c82254db56aec22 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Tue, 8 Mar 2022 16:27:34 +0000 Subject: [PATCH 09/26] docs: Update contact link in runtime README This PR updates the contact link in the runtime README document. Fixes #3854 Signed-off-by: Gabriela Cervantes --- src/runtime/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/README.md b/src/runtime/README.md index fcffc238ad..0cd77b9956 100644 --- a/src/runtime/README.md +++ b/src/runtime/README.md @@ -158,7 +158,7 @@ See [the community repository](https://github.com/kata-containers/community). ### Contact -See [how to reach the community](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md#contact). +See [how to reach the community](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md#contact). ## Further information From 4b1e2f527eb98cd454bd1b3d5e55c7788d9aace6 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 8 Mar 2022 17:06:16 +0000 Subject: [PATCH 10/26] CI: Update GHA secret name Change the secret used by the GitHub Action that adds the PR size label to one with the correct set of privileges. Fixes: #3856. Signed-off-by: James O. D. Hunt --- .github/workflows/add-pr-sizing-label.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/add-pr-sizing-label.yaml b/.github/workflows/add-pr-sizing-label.yaml index 751a942762..0a2d7fcbf7 100644 --- a/.github/workflows/add-pr-sizing-label.yaml +++ b/.github/workflows/add-pr-sizing-label.yaml @@ -30,7 +30,7 @@ jobs: - name: Add PR sizing label env: - GITHUB_TOKEN: ${{ secrets.KATA_GITHUB_ACTIONS_TOKEN }} + GITHUB_TOKEN: ${{ secrets.KATA_GITHUB_ACTIONS_PR_SIZE_TOKEN }} run: | pr=${{ github.event.number }} sudo apt -y install diffstat patchutils From c088a3f3adcc4b8ff1134bed9335aca31a3d131e Mon Sep 17 00:00:00 2001 From: Braden Rayhorn Date: Sat, 5 Mar 2022 18:32:34 -0600 Subject: [PATCH 11/26] agent: add tests for get_memory_info function Add test coverage for get_memory_info function in src/rpc.rs. Includes some minor refactoring of the function. Fixes #3837 Signed-off-by: Braden Rayhorn --- src/agent/src/rpc.rs | 170 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 161 insertions(+), 9 deletions(-) diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 1f4729b7d0..2325347160 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -85,6 +85,8 @@ use std::path::PathBuf; const CONTAINER_BASE: &str = "/run/kata-containers"; const MODPROBE_PATH: &str = "/sbin/modprobe"; +const ERR_INVALID_BLOCK_SIZE: &str = "Invalid block size"; + // Convenience macro to obtain the scope logger macro_rules! sl { () => { @@ -1154,7 +1156,12 @@ impl protocols::agent_ttrpc::AgentService for AgentService { info!(sl!(), "get guest details!"); let mut resp = GuestDetailsResponse::new(); // to get memory block size - match get_memory_info(req.mem_block_size, req.mem_hotplug_probe) { + match get_memory_info( + req.mem_block_size, + req.mem_hotplug_probe, + SYSFS_MEMORY_BLOCK_SIZE_PATH, + SYSFS_MEMORY_HOTPLUG_PROBE_PATH, + ) { Ok((u, v)) => { resp.mem_block_size_bytes = u; resp.support_mem_hotplug_probe = v; @@ -1343,24 +1350,29 @@ impl protocols::health_ttrpc::Health for HealthService { } } -fn get_memory_info(block_size: bool, hotplug: bool) -> Result<(u64, bool)> { +fn get_memory_info( + block_size: bool, + hotplug: bool, + block_size_path: &str, + hotplug_probe_path: &str, +) -> Result<(u64, bool)> { let mut size: u64 = 0; let mut plug: bool = false; if block_size { - match fs::read_to_string(SYSFS_MEMORY_BLOCK_SIZE_PATH) { + match fs::read_to_string(block_size_path) { Ok(v) => { if v.is_empty() { - info!(sl!(), "string in empty???"); - return Err(anyhow!("Invalid block size")); + warn!(sl!(), "file {} is empty", block_size_path); + return Err(anyhow!(ERR_INVALID_BLOCK_SIZE)); } size = u64::from_str_radix(v.trim(), 16).map_err(|_| { warn!(sl!(), "failed to parse the str {} to hex", size); - anyhow!("Invalid block size") + anyhow!(ERR_INVALID_BLOCK_SIZE) })?; } Err(e) => { - info!(sl!(), "memory block size error: {:?}", e.kind()); + warn!(sl!(), "memory block size error: {:?}", e.kind()); if e.kind() != std::io::ErrorKind::NotFound { return Err(anyhow!(e)); } @@ -1369,10 +1381,10 @@ fn get_memory_info(block_size: bool, hotplug: bool) -> Result<(u64, bool)> { } if hotplug { - match stat::stat(SYSFS_MEMORY_HOTPLUG_PROBE_PATH) { + match stat::stat(hotplug_probe_path) { Ok(_) => plug = true, Err(e) => { - info!(sl!(), "hotplug memory error: {:?}", e); + warn!(sl!(), "hotplug memory error: {:?}", e); match e { nix::Error::ENOENT => plug = false, _ => return Err(anyhow!(e)), @@ -1803,8 +1815,35 @@ mod tests { use super::*; use crate::protocols::agent_ttrpc::AgentService as _; use oci::{Hook, Hooks}; + use tempfile::tempdir; use ttrpc::{r#async::TtrpcContext, MessageHeader}; + // Parameters: + // + // 1: expected Result + // 2: actual Result + // 3: string used to identify the test on error + macro_rules! assert_result { + ($expected_result:expr, $actual_result:expr, $msg:expr) => { + if $expected_result.is_ok() { + let expected_level = $expected_result.as_ref().unwrap(); + let actual_level = $actual_result.unwrap(); + assert!(*expected_level == actual_level, "{}", $msg); + } else { + let expected_error = $expected_result.as_ref().unwrap_err(); + let expected_error_msg = format!("{:?}", expected_error); + + if let Err(actual_error) = $actual_result { + let actual_error_msg = format!("{:?}", actual_error); + + assert!(expected_error_msg == actual_error_msg, "{}", $msg); + } else { + assert!(expected_error_msg == "expected error, got OK", "{}", $msg); + } + } + }; + } + fn mk_ttrpc_context() -> TtrpcContext { TtrpcContext { fd: -1, @@ -1906,6 +1945,119 @@ mod tests { assert!(result.is_err(), "expected add arp neighbors to fail"); } + #[tokio::test] + async fn test_get_memory_info() { + #[derive(Debug)] + struct TestData<'a> { + // if None is provided, no file will be generated, else the data in the Option will populate the file + block_size_data: Option<&'a str>, + + hotplug_probe_data: bool, + get_block_size: bool, + get_hotplug: bool, + result: Result<(u64, bool)>, + } + + let tests = &[ + TestData { + block_size_data: Some("10000000"), + hotplug_probe_data: true, + get_block_size: true, + get_hotplug: true, + result: Ok((268435456, true)), + }, + TestData { + block_size_data: Some("100"), + hotplug_probe_data: false, + get_block_size: true, + get_hotplug: true, + result: Ok((256, false)), + }, + TestData { + block_size_data: None, + hotplug_probe_data: false, + get_block_size: true, + get_hotplug: true, + result: Ok((0, false)), + }, + TestData { + block_size_data: Some(""), + hotplug_probe_data: false, + get_block_size: true, + get_hotplug: false, + result: Err(anyhow!(ERR_INVALID_BLOCK_SIZE)), + }, + TestData { + block_size_data: Some("-1"), + hotplug_probe_data: false, + get_block_size: true, + get_hotplug: false, + result: Err(anyhow!(ERR_INVALID_BLOCK_SIZE)), + }, + TestData { + block_size_data: Some(" "), + hotplug_probe_data: false, + get_block_size: true, + get_hotplug: false, + result: Err(anyhow!(ERR_INVALID_BLOCK_SIZE)), + }, + TestData { + block_size_data: Some("some data"), + hotplug_probe_data: false, + get_block_size: true, + get_hotplug: false, + result: Err(anyhow!(ERR_INVALID_BLOCK_SIZE)), + }, + TestData { + block_size_data: Some("some data"), + hotplug_probe_data: true, + get_block_size: false, + get_hotplug: false, + result: Ok((0, false)), + }, + TestData { + block_size_data: Some("100"), + hotplug_probe_data: true, + get_block_size: false, + get_hotplug: false, + result: Ok((0, false)), + }, + TestData { + block_size_data: Some("100"), + hotplug_probe_data: true, + get_block_size: false, + get_hotplug: true, + result: Ok((0, true)), + }, + ]; + + for (i, d) in tests.iter().enumerate() { + let msg = format!("test[{}]: {:?}", i, d); + + let dir = tempdir().expect("failed to make tempdir"); + let block_size_path = dir.path().join("block_size_bytes"); + let hotplug_probe_path = dir.path().join("probe"); + + if let Some(block_size_data) = d.block_size_data { + fs::write(&block_size_path, block_size_data).unwrap(); + } + if d.hotplug_probe_data { + fs::write(&hotplug_probe_path, []).unwrap(); + } + + let result = get_memory_info( + d.get_block_size, + d.get_hotplug, + block_size_path.to_str().unwrap(), + hotplug_probe_path.to_str().unwrap(), + ); + + let msg = format!("{}, result: {:?}", msg, result); + + assert_result!(d.result, result, msg); + } + } + #[tokio::test] async fn test_verify_cid() { #[derive(Debug)] From d234cb76b5c2bff9c454e12044ee18f0bbe4b15b Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 8 Mar 2022 16:29:36 +0000 Subject: [PATCH 12/26] manager: Create containerd link Make the `kata-manager` create a `containerd` link to ensure the downloaded containerd systemd service file can find the daemon when using the GitHub packaged version of containerd. Signed-off-by: James O. D. Hunt --- utils/kata-manager.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/utils/kata-manager.sh b/utils/kata-manager.sh index 38260c865b..9c298c9eb3 100755 --- a/utils/kata-manager.sh +++ b/utils/kata-manager.sh @@ -402,7 +402,12 @@ install_containerd() sudo tar -C /usr/local -xvf "${file}" - sudo ln -sf /usr/local/bin/ctr "${link_dir}" + for file in \ + /usr/local/bin/containerd \ + /usr/local/bin/ctr + do + sudo ln -sf "$file" "${link_dir}" + done info "$project installed\n" } From 019da91d7996f1b4284d8f20e15d281e3ea309b7 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 8 Mar 2022 16:30:28 +0000 Subject: [PATCH 13/26] manager: Whitespace fix Remove additional blank line in the `kata-manager`. Signed-off-by: James O. D. Hunt --- utils/kata-manager.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/kata-manager.sh b/utils/kata-manager.sh index 9c298c9eb3..d928942d70 100755 --- a/utils/kata-manager.sh +++ b/utils/kata-manager.sh @@ -441,7 +441,6 @@ configure_containerd() "$(date -Iseconds)" |\ tee -a "$containerd_service_name" - sudo cp "${containerd_service_name}" "${dest}" sudo systemctl daemon-reload From d4d65bed387152c02c6c74625998377315d86410 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 8 Mar 2022 16:37:39 +0000 Subject: [PATCH 14/26] manager: Add option to enable component debug Added a `-d` option to `kata-manager` to enable Kata Containers and containerd debug. Signed-off-by: James O. D. Hunt --- utils/kata-manager.sh | 116 +++++++++++++++++++++++++++++++++++------- 1 file changed, 97 insertions(+), 19 deletions(-) diff --git a/utils/kata-manager.sh b/utils/kata-manager.sh index d928942d70..94064150f3 100755 --- a/utils/kata-manager.sh +++ b/utils/kata-manager.sh @@ -208,6 +208,7 @@ Description: Install $kata_project [1] (and optionally $containerd_project [2]) Options: -c : Specify containerd version. + -d : Enable debug for all components. -f : Force installation (use with care). -h : Show this help statement. -k : Specify Kata Containers version. @@ -414,6 +415,9 @@ install_containerd() configure_containerd() { + local enable_debug="${1:-}" + [ -z "$enable_debug" ] && die "no enable debug value" + local project="$containerd_project" info "Configuring $project" @@ -464,26 +468,55 @@ configure_containerd() info "Backed up $cfg to $original" } + local modified="false" + # Add the Kata Containers configuration details: + local comment_text + comment_text=$(printf "%s: Added by %s\n" \ + "$(date -Iseconds)" \ + "$script_name") + sudo grep -q "$kata_runtime_type" "$cfg" || { cat <<-EOT | sudo tee -a "$cfg" -[plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - default_runtime_name = "${kata_runtime_name}" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.${kata_runtime_name}] - runtime_type = "${kata_runtime_type}" -EOT + # $comment_text + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "${kata_runtime_name}" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.${kata_runtime_name}] + runtime_type = "${kata_runtime_type}" + EOT - info "Modified $cfg" + modified="true" } + if [ "$enable_debug" = "true" ] + then + local debug_enabled + debug_enabled=$(awk -v RS='' '/\[debug\]/' "$cfg" |\ + grep -E "^\s*\\s*=\s*.*\" || true) + + [ -n "$debug_enabled" ] || { + cat <<-EOT | sudo tee -a "$cfg" + # $comment_text + [debug] + level = "debug" + EOT + } + + modified="true" + fi + + [ "$modified" = "true" ] && info "Modified $cfg" sudo systemctl enable containerd sudo systemctl start containerd - info "Configured $project\n" + local msg="disabled" + [ "$enable_debug" = "true" ] && msg="enabled" + + info "Configured $project (debug $msg)\n" } install_kata() @@ -544,11 +577,48 @@ install_kata() info "$project installed\n" } +configure_kata() +{ + local enable_debug="${1:-}" + [ -z "$enable_debug" ] && die "no enable debug value" + + [ "$enable_debug" = "false" ] && \ + info "Using default $kata_project configuration" && \ + return 0 + + local config_file='configuration.toml' + local kata_dir='/etc/kata-containers' + + sudo mkdir -p "$kata_dir" + + local cfg_from + local cfg_to + + cfg_from="${kata_install_dir}/share/defaults/kata-containers/${config_file}" + cfg_to="${kata_dir}/${config_file}" + + [ -e "$cfg_from" ] || die "cannot find $kata_project configuration file" + + sudo install -o root -g root -m 0644 "$cfg_from" "$cfg_to" + + sudo sed -i \ + -e 's/^# *\(enable_debug\).*=.*$/\1 = true/g' \ + -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.log=debug initcall_debug"/g' \ + "$cfg_to" + + info "Configured $kata_project for full debug (delete $cfg_to to use pristine $kata_project configuration)" +} + handle_kata() { local version="${1:-}" - install_kata "$version" + local enable_debug="${2:-}" + [ -z "$enable_debug" ] && die "no enable debug value" + + install_kata "$version" "$enable_debug" + + configure_kata "$enable_debug" kata-runtime --version } @@ -560,6 +630,9 @@ handle_containerd() local force="${2:-}" [ -z "$force" ] && die "need force value" + local enable_debug="${3:-}" + [ -z "$enable_debug" ] && die "no enable debug value" + local ret if [ "$force" = "true" ] @@ -576,7 +649,7 @@ handle_containerd() fi fi - configure_containerd + configure_containerd "$enable_debug" containerd --version } @@ -621,20 +694,22 @@ handle_installation() local only_kata="${3:-}" [ -z "$only_kata" ] && die "no only Kata value" + local enable_debug="${4:-}" + [ -z "$enable_debug" ] && die "no enable debug value" + # These params can be blank - local kata_version="${4:-}" - local containerd_version="${5:-}" + local kata_version="${5:-}" + local containerd_version="${6:-}" setup "$cleanup" "$force" - handle_kata "$kata_version" + handle_kata "$kata_version" "$enable_debug" [ "$only_kata" = "false" ] && \ handle_containerd \ "$containerd_version" \ - "$force" - - test_installation + "$force" \ + "$enable_debug" if [ "$only_kata" = "true" ] then @@ -651,16 +726,18 @@ handle_args() local cleanup="true" local force="false" local only_kata="false" + local enable_debug="false" local opt local kata_version="" local containerd_version="" - while getopts "c:fhk:or" opt "$@" + while getopts "c:dfhk:or" opt "$@" do case "$opt" in c) containerd_version="$OPTARG" ;; + d) enable_debug="true" ;; f) force="true" ;; h) usage; exit 0 ;; k) kata_version="$OPTARG" ;; @@ -678,6 +755,7 @@ handle_args() "$cleanup" \ "$force" \ "$only_kata" \ + "$enable_debug" \ "$kata_version" \ "$containerd_version" } From 9576a7da5d5c39e7ea5e20adade43dd6f1f1ebcc Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Tue, 8 Mar 2022 16:41:54 +0000 Subject: [PATCH 15/26] manager: Add options to change self test behaviour Added new `kata-manager` options to control the self-test behaviour. By default, after installation the manager will run a test to ensure a Kata Containers container can be created. New options allow: - The self test to be disabled. - Only the self test to be run (no installation). These features allow changes to be made to the installed system before the self test is run. Fixes: #3851. Signed-off-by: James O. D. Hunt --- utils/kata-manager.sh | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/utils/kata-manager.sh b/utils/kata-manager.sh index 94064150f3..630447c9a6 100755 --- a/utils/kata-manager.sh +++ b/utils/kata-manager.sh @@ -214,6 +214,8 @@ Options: -k : Specify Kata Containers version. -o : Only install Kata Containers. -r : Don't cleanup on failure (retain files). + -t : Disable self test (don't try to create a container after install). + -T : Only run self test (do not install anything). Notes: @@ -697,9 +699,17 @@ handle_installation() local enable_debug="${4:-}" [ -z "$enable_debug" ] && die "no enable debug value" + local disable_test="${5:-}" + [ -z "$disable_test" ] && die "no disable test value" + + local only_run_test="${6:-}" + [ -z "$only_run_test" ] && die "no only run test value" + # These params can be blank - local kata_version="${5:-}" - local containerd_version="${6:-}" + local kata_version="${7:-}" + local containerd_version="${8:-}" + + [ "$only_run_test" = "true" ] && test_installation && return 0 setup "$cleanup" "$force" @@ -711,6 +721,8 @@ handle_installation() "$force" \ "$enable_debug" + [ "$disable_test" = "false" ] && test_installation + if [ "$only_kata" = "true" ] then info "$kata_project is now installed" @@ -726,6 +738,8 @@ handle_args() local cleanup="true" local force="false" local only_kata="false" + local disable_test="false" + local only_run_test="false" local enable_debug="false" local opt @@ -733,7 +747,7 @@ handle_args() local kata_version="" local containerd_version="" - while getopts "c:dfhk:or" opt "$@" + while getopts "c:dfhk:ortT" opt "$@" do case "$opt" in c) containerd_version="$OPTARG" ;; @@ -743,6 +757,8 @@ handle_args() k) kata_version="$OPTARG" ;; o) only_kata="true" ;; r) cleanup="false" ;; + t) disable_test="true" ;; + T) only_run_test="true" ;; esac done @@ -756,6 +772,8 @@ handle_args() "$force" \ "$only_kata" \ "$enable_debug" \ + "$disable_test" \ + "$only_run_test" \ "$kata_version" \ "$containerd_version" } From be12baf3cfdc2ded1b4bea01ff874f4e319e4a8e Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 10 Mar 2022 09:18:40 +0000 Subject: [PATCH 16/26] manager: Change here documents to use standard delimiter All scripts should use `EOF` as the shell here document delimiter as this is checked by the static checker. Signed-off-by: James O. D. Hunt --- utils/kata-manager.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/utils/kata-manager.sh b/utils/kata-manager.sh index 630447c9a6..1cb65145fb 100755 --- a/utils/kata-manager.sh +++ b/utils/kata-manager.sh @@ -43,7 +43,7 @@ readonly link_dir=${link_dir:-/usr/bin} readonly tmpdir=$(mktemp -d) -readonly warnings=$(cat < []] Description: Install $kata_project [1] (and optionally $containerd_project [2]) @@ -235,7 +235,7 @@ Advice: $ kata-runtime check --only-list-releases -EOT +EOF } # Determine if the system only supports cgroups v2. @@ -480,7 +480,7 @@ configure_containerd() "$script_name") sudo grep -q "$kata_runtime_type" "$cfg" || { - cat <<-EOT | sudo tee -a "$cfg" + cat <<-EOF | sudo tee -a "$cfg" # $comment_text [plugins] [plugins."io.containerd.grpc.v1.cri"] @@ -489,7 +489,7 @@ configure_containerd() [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.${kata_runtime_name}] runtime_type = "${kata_runtime_type}" - EOT + EOF modified="true" } @@ -501,11 +501,11 @@ configure_containerd() grep -E "^\s*\\s*=\s*.*\" || true) [ -n "$debug_enabled" ] || { - cat <<-EOT | sudo tee -a "$cfg" + cat <<-EOF | sudo tee -a "$cfg" # $comment_text [debug] level = "debug" - EOT + EOF } modified="true" From 5d6d39be48189124e9c7823d860e9578204de649 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Thu, 10 Mar 2022 09:23:37 +0000 Subject: [PATCH 17/26] scripts: Change here document delimiters Fix the outstanding scripts using non standard shell here document delimiters. This should have been caught by https://github.com/kata-containers/tests/pull/3937, but there is a bug in the checker which is fixed on https://github.com/kata-containers/tests/pull/4569. Fixes: #3864. Signed-off-by: James O. D. Hunt --- .../protocols/hack/update-generated-proto.sh | 4 ++-- tools/osbuilder/image-builder/image_builder.sh | 4 ++-- tools/osbuilder/initrd-builder/initrd_builder.sh | 4 ++-- tools/osbuilder/rootfs-builder/rootfs.sh | 12 ++++++------ tools/osbuilder/scripts/lib.sh | 4 ++-- tools/osbuilder/tests/test_images.sh | 4 ++-- tools/packaging/guest-image/build_image.sh | 4 ++-- .../local-build/kata-deploy-binaries.sh | 4 ++-- .../packaging/kata-deploy/scripts/kata-deploy.sh | 16 ++++++++-------- tools/packaging/kernel/build-kernel.sh | 4 ++-- tools/packaging/release/publish-kata-image.sh | 4 ++-- tools/packaging/release/release-notes.sh | 16 ++++++++-------- tools/packaging/release/tag_repos.sh | 8 ++++---- .../release/update-repository-version.sh | 8 ++++---- tools/packaging/scripts/apply_patches.sh | 4 ++-- tools/packaging/scripts/configure-hypervisor.sh | 8 ++++---- tools/packaging/scripts/gen_versions_txt.sh | 8 ++++---- .../packaging/static-build/shim-v2/install_go.sh | 4 ++-- 18 files changed, 60 insertions(+), 60 deletions(-) diff --git a/src/libs/protocols/hack/update-generated-proto.sh b/src/libs/protocols/hack/update-generated-proto.sh index 64952b03c4..67f25058aa 100755 --- a/src/libs/protocols/hack/update-generated-proto.sh +++ b/src/libs/protocols/hack/update-generated-proto.sh @@ -7,14 +7,14 @@ # // die() { - cat <&2 + cat <&2 ==================================================================== ==== compile protocols failed ==== $1 ==================================================================== -EOT +EOF exit 1 } diff --git a/tools/osbuilder/image-builder/image_builder.sh b/tools/osbuilder/image-builder/image_builder.sh index 5d01b0a2d8..8b65ab4e8f 100755 --- a/tools/osbuilder/image-builder/image_builder.sh +++ b/tools/osbuilder/image-builder/image_builder.sh @@ -72,7 +72,7 @@ readonly mem_boundary_mb=128 source "${lib_file}" usage() { - cat < This script will create a Kata Containers image file of an adequate size based on the directory. @@ -117,7 +117,7 @@ Kernels and hypervisors that support DAX/NVDIMM read the MBR #2, otherwise MBR # [1] - https://github.com/kata-containers/kata-containers/blob/main/tools/osbuilder/image-builder/nsdax.gpl.c [2] - https://github.com/torvalds/linux/blob/master/drivers/nvdimm/pfn.h -EOT +EOF } diff --git a/tools/osbuilder/initrd-builder/initrd_builder.sh b/tools/osbuilder/initrd-builder/initrd_builder.sh index 1d4a60d524..fb95f6a38c 100755 --- a/tools/osbuilder/initrd-builder/initrd_builder.sh +++ b/tools/osbuilder/initrd-builder/initrd_builder.sh @@ -23,7 +23,7 @@ AGENT_INIT=${AGENT_INIT:-no} usage() { error="${1:-0}" - cat < This script creates a Kata Containers initrd image file based on the directory. @@ -38,7 +38,7 @@ Extra environment variables: DEFAULT: kata-agent AGENT_INIT: use kata agent as init process DEFAULT: no -EOT +EOF exit "${error}" } diff --git a/tools/osbuilder/rootfs-builder/rootfs.sh b/tools/osbuilder/rootfs-builder/rootfs.sh index 80633a0451..a7d1a0a3be 100755 --- a/tools/osbuilder/rootfs-builder/rootfs.sh +++ b/tools/osbuilder/rootfs-builder/rootfs.sh @@ -63,7 +63,7 @@ typeset init= usage() { error="${1:-0}" - cat < "$unitFile" << EOT + cp ./usr/share/systemd/tmp.mount "$unitFile" || cat > "$unitFile" << EOF # This file is part of systemd. # # systemd is free software; you can redistribute it and/or modify it @@ -496,7 +496,7 @@ What=tmpfs Where=/tmp Type=tmpfs Options=mode=1777,strictatime,nosuid,nodev -EOT +EOF fi popd >> /dev/null @@ -519,12 +519,12 @@ EOT esac info "Configure chrony file ${chrony_conf_file}" - cat >> "${chrony_conf_file}" <> "${chrony_conf_file}" <"$file"<<-EOT + cat >"$file"<<-EOF --- osbuilder: url: "${osbuilder_url}" @@ -190,7 +190,7 @@ ${extra} name: "${AGENT_BIN}" version: "${agent_version}" agent-is-init-daemon: "${AGENT_INIT}" -EOT +EOF local rootfs_file="${file_dir}/$(basename "${file}")" info "Created summary file '${rootfs_file}' inside rootfs" diff --git a/tools/osbuilder/tests/test_images.sh b/tools/osbuilder/tests/test_images.sh index fbbe5d5901..5def439b7c 100755 --- a/tools/osbuilder/tests/test_images.sh +++ b/tools/osbuilder/tests/test_images.sh @@ -53,7 +53,7 @@ source "${project_dir}/scripts/lib.sh" usage() { - cat <] Options: @@ -72,7 +72,7 @@ Otherwise, tests are run on all distros. $(basename ${test_config}) includes a list of distros to exclude from testing, depending on the detected test environment. However, when a is specified, distro exclusion based on $(basename ${test_config}) is not enforced. -EOT +EOF } # Add an entry to the specified stats file diff --git a/tools/packaging/guest-image/build_image.sh b/tools/packaging/guest-image/build_image.sh index d3dc0c54d7..3913df51d1 100755 --- a/tools/packaging/guest-image/build_image.sh +++ b/tools/packaging/guest-image/build_image.sh @@ -59,7 +59,7 @@ build_image() { usage() { return_code=${1:-0} - cat < : Hypervisor_target. -v : Kernel version to use if kernel path not provided. -x : Confidential guest protection type, such as sev and tdx -EOT +EOF exit "$exit_code" } diff --git a/tools/packaging/release/publish-kata-image.sh b/tools/packaging/release/publish-kata-image.sh index 49613ac82b..1240e4f5dd 100755 --- a/tools/packaging/release/publish-kata-image.sh +++ b/tools/packaging/release/publish-kata-image.sh @@ -28,7 +28,7 @@ die() { usage() { return_code=${1:-0} - cat < @@ -41,7 +41,7 @@ options: -h : show this help -p : push image to github -EOT +EOF exit "${return_code}" } diff --git a/tools/packaging/release/release-notes.sh b/tools/packaging/release/release-notes.sh index acf543bcfb..4b7019466a 100755 --- a/tools/packaging/release/release-notes.sh +++ b/tools/packaging/release/release-notes.sh @@ -26,7 +26,7 @@ trap exit_handler EXIT usage() { return_code=${1:-} - cat < Args: @@ -37,7 +37,7 @@ new-release: new release version that will have the Example: ./${script_name} 1.2.0 1.2.1 > notes.md -EOT +EOF exit "${return_code}" } @@ -81,25 +81,25 @@ changes() { } print_release_notes() { - cat <>/dev/null - cat <>/dev/null rm -rf "${tmp_dir}/${repo}" done - cat < This script creates a new release for ${PROJECT}. It tags and create release for: -EOT +EOF for r in "${repos[@]}"; do echo " - ${r}" done - cat <"${notes_file}" + cat <"${notes_file}" # Kata Containers ${new_version} $(get_changes "$current_version") -EOT +EOF cat "${notes_file}" if (echo "${current_version}" | grep "alpha") && (echo "${new_version}" | grep -v "alpha");then @@ -282,7 +282,7 @@ EOT usage() { exit_code="$1" - cat < Args: @@ -293,7 +293,7 @@ Example: Options -h : Show this help -p : create a PR -EOT +EOF exit "$exit_code" } diff --git a/tools/packaging/scripts/apply_patches.sh b/tools/packaging/scripts/apply_patches.sh index de85808451..e5092779f5 100755 --- a/tools/packaging/scripts/apply_patches.sh +++ b/tools/packaging/scripts/apply_patches.sh @@ -12,7 +12,7 @@ script_dir="$(realpath $(dirname $0))" patches_dir="$1" if [ -z "$patches_dir" ]; then - cat <<-EOT + cat <<-EOF Apply patches to the sources at the current directory. Patches are expected to be named in the standard git-format-patch(1) format where @@ -30,7 +30,7 @@ if [ -z "$patches_dir" ]; then $0 PATCHES_DIR Where: PATCHES_DIR is the directory containing the patches - EOT + EOF exit 1 fi diff --git a/tools/packaging/scripts/configure-hypervisor.sh b/tools/packaging/scripts/configure-hypervisor.sh index 4db23ae663..1d3689927e 100755 --- a/tools/packaging/scripts/configure-hypervisor.sh +++ b/tools/packaging/scripts/configure-hypervisor.sh @@ -72,7 +72,7 @@ die() { # Display usage to stdout. usage() { - cat < "$versions_txt" < "$versions_txt" < @@ -98,7 +98,7 @@ Options: --compare Only compare the kata version at branch with the one in ${versions_txt} and leave the file untouched. --head Use 's head to generate the versions file. -EOT +EOF exit "${exit_code}" } diff --git a/tools/packaging/static-build/shim-v2/install_go.sh b/tools/packaging/static-build/shim-v2/install_go.sh index cf4b311c41..4872dc49cf 100755 --- a/tools/packaging/static-build/shim-v2/install_go.sh +++ b/tools/packaging/static-build/shim-v2/install_go.sh @@ -31,7 +31,7 @@ info() { usage(){ exit_code="$1" - cat < : destination path, path where go will be installed. -f : enable force install, remove existent go pkg before installation. -h : display this help. -EOT +EOF exit "$exit_code" } From 5c434270d14734455aad133624a7ab5379cf7be8 Mon Sep 17 00:00:00 2001 From: Chelsea Mafrica Date: Wed, 9 Mar 2022 17:30:41 -0800 Subject: [PATCH 18/26] docs: Update k8s documentation Update documentation with missing step to untaint node to enable scheduling and update the example to run a pod using the kata runtime class instead of untrusted workloads, which applies to versions of CRI-O prior to v1.12. Fixes #3863 Signed-off-by: Chelsea Mafrica --- docs/how-to/run-kata-with-k8s.md | 77 +++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 17 deletions(-) diff --git a/docs/how-to/run-kata-with-k8s.md b/docs/how-to/run-kata-with-k8s.md index 29d7309876..fd53838b88 100644 --- a/docs/how-to/run-kata-with-k8s.md +++ b/docs/how-to/run-kata-with-k8s.md @@ -104,26 +104,69 @@ $ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /run/containerd/c $ export KUBECONFIG=/etc/kubernetes/admin.conf ``` -You can force Kubelet to use Kata Containers by adding some `untrusted` -annotation to your pod configuration. In our case, this ensures Kata -Containers is the selected runtime to run the described workload. +### Allow pods to run in the master node -`nginx-untrusted.yaml` -```yaml -apiVersion: v1 -kind: Pod +By default, the cluster will not schedule pods in the master node. To enable master node scheduling: +```bash +$ sudo -E kubectl taint nodes --all node-role.kubernetes.io/master- +``` + +### Create runtime class for Kata Containers + +Users can use [`RuntimeClass`](https://kubernetes.io/docs/concepts/containers/runtime-class/#runtime-class) to specify a different runtime for Pods. + +```bash +$ cat > runtime.yaml < Date: Fri, 11 Mar 2022 08:27:08 -0800 Subject: [PATCH 19/26] versions: Upgrade to Cloud Hypervisor v22.1 This is a bug fix release. The following issues have been addressed: 1) VFIO ioctl reordering to fix MSI on AMD platforms; 2) Fix virtio-net control queue. Details can be found: https://github.com/cloud-hypervisor/cloud-hypervisor/releases/tag/v22.1 Fixes: #3872 Signed-off-by: Bo Chen --- versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.yaml b/versions.yaml index 11badc6eb5..3384287432 100644 --- a/versions.yaml +++ b/versions.yaml @@ -75,7 +75,7 @@ assets: url: "https://github.com/cloud-hypervisor/cloud-hypervisor" uscan-url: >- https://github.com/cloud-hypervisor/cloud-hypervisor/tags.*/v?(\d\S+)\.tar\.gz - version: "v22.0" + version: "v22.1" firecracker: description: "Firecracker micro-VMM" From 92ce5e2dc4766b0ce95e0b8d72bc1344033c212d Mon Sep 17 00:00:00 2001 From: Jack Hance Date: Thu, 10 Mar 2022 13:59:59 -0600 Subject: [PATCH 20/26] rustjail: optimization, merged several writelns into one Optimized several writelns by merging them into one in src/utils.rs Fixes: #3772 Signed-off-by: Jack Hance --- src/agent/rustjail/src/utils.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/agent/rustjail/src/utils.rs b/src/agent/rustjail/src/utils.rs index 60c3d25b1c..cf69c77a2a 100644 --- a/src/agent/rustjail/src/utils.rs +++ b/src/agent/rustjail/src/utils.rs @@ -97,12 +97,13 @@ mod tests { let temp_passwd = format!("{}/passwd", tmpdir_path); let mut tempf = File::create(temp_passwd.as_str()).unwrap(); - writeln!(tempf, "root:x:0:0:root:/root0:/bin/bash").unwrap(); - writeln!(tempf, "root:x:1:0:root:/root1:/bin/bash").unwrap(); - writeln!(tempf, "#root:x:1:0:root:/rootx:/bin/bash").unwrap(); - writeln!(tempf, "root:x:2:0:root:/root2:/bin/bash").unwrap(); - writeln!(tempf, "root:x:3:0:root:/root3").unwrap(); - writeln!(tempf, "root:x:3:0:root:/root3:/bin/bash").unwrap(); + let passwd_entries = "root:x:0:0:root:/root0:/bin/bash +root:x:1:0:root:/root1:/bin/bash +#root:x:1:0:root:/rootx:/bin/bash +root:x:2:0:root:/root2:/bin/bash +root:x:3:0:root:/root3 +root:x:3:0:root:/root3:/bin/bash"; + writeln!(tempf, "{}", passwd_entries).unwrap(); let entry = get_entry_by_uid(0, temp_passwd.as_str()).unwrap(); assert_eq!(entry.dir.as_str(), "/root0"); From dacf6e395567b3be914c8c06ea7445d1e27e033d Mon Sep 17 00:00:00 2001 From: Garrett Mahin Date: Sun, 13 Mar 2022 17:39:08 -0500 Subject: [PATCH 21/26] doc: fix filename typo Corrects a filename typo in cleanup cluster part of kata-deploy README.md Fixes: #3869 Signed-off-by: Garrett Mahin --- tools/packaging/kata-deploy/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/packaging/kata-deploy/README.md b/tools/packaging/kata-deploy/README.md index b6d455adc8..c298c61454 100644 --- a/tools/packaging/kata-deploy/README.md +++ b/tools/packaging/kata-deploy/README.md @@ -143,7 +143,7 @@ $ kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod After ensuring kata-deploy has been deleted, cleanup the cluster: ```sh -$ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stabe.yaml +$ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup-stable.yaml ``` The cleanup daemon-set will run a single time, cleaning up the node-label, which makes it difficult to check in an automated fashion. From efa19c41ebede954ae9c2ead97dd970691632f91 Mon Sep 17 00:00:00 2001 From: zhanghj Date: Mon, 14 Mar 2022 09:17:04 +0800 Subject: [PATCH 22/26] device: use const strings for block-driver option instead of hard coding Currently, the block driver option is specifed by hard coding, maybe it is better to use const string variables instead of hard coded strings. Another modification is to remove duplicate consts for virtio driver in manager.go. Fixes: #3321 Signed-off-by: Jason Zhang --- src/runtime/virtcontainers/container_test.go | 4 +-- .../virtcontainers/device/config/config.go | 10 ++++-- .../virtcontainers/device/config/pmem.go | 4 +-- .../virtcontainers/device/drivers/block.go | 16 ++++----- .../device/drivers/vhost_user_blk.go | 10 +++--- .../virtcontainers/device/manager/manager.go | 35 ++++++------------- .../device/manager/manager_linux_test.go | 2 +- .../device/manager/manager_test.go | 12 +++---- .../documentation/api/1.0/api.md | 4 +-- src/runtime/virtcontainers/kata_agent_test.go | 4 +-- .../virtcontainers/persist/api/device.go | 2 +- src/runtime/virtcontainers/persist_test.go | 6 ++-- src/runtime/virtcontainers/sandbox_test.go | 4 +-- 13 files changed, 53 insertions(+), 60 deletions(-) diff --git a/src/runtime/virtcontainers/container_test.go b/src/runtime/virtcontainers/container_test.go index b41fcc1089..eddf8ed701 100644 --- a/src/runtime/virtcontainers/container_test.go +++ b/src/runtime/virtcontainers/container_test.go @@ -86,7 +86,7 @@ func TestContainerRemoveDrive(t *testing.T) { sandbox := &Sandbox{ ctx: context.Background(), id: "sandbox", - devManager: manager.NewDeviceManager(manager.VirtioSCSI, false, "", nil), + devManager: manager.NewDeviceManager(config.VirtioSCSI, false, "", nil), config: &SandboxConfig{}, } @@ -320,7 +320,7 @@ func TestContainerAddDriveDir(t *testing.T) { sandbox := &Sandbox{ ctx: context.Background(), id: testSandboxID, - devManager: manager.NewDeviceManager(manager.VirtioSCSI, false, "", nil), + devManager: manager.NewDeviceManager(config.VirtioSCSI, false, "", nil), hypervisor: &mockHypervisor{}, agent: &mockAgent{}, config: &SandboxConfig{ diff --git a/src/runtime/virtcontainers/device/config/config.go b/src/runtime/virtcontainers/device/config/config.go index 69be4f5832..48280092d6 100644 --- a/src/runtime/virtcontainers/device/config/config.go +++ b/src/runtime/virtcontainers/device/config/config.go @@ -51,7 +51,7 @@ const ( // VirtioBlock means use virtio-blk for hotplugging drives VirtioBlock = "virtio-blk" - // VirtioBlockCCW means use virtio-blk for hotplugging drives + // VirtioBlockCCW means use virtio-blk-ccw for hotplugging drives VirtioBlockCCW = "virtio-blk-ccw" // VirtioSCSI means use virtio-scsi for hotplugging drives @@ -72,6 +72,12 @@ const ( VirtioFSNydus = "virtio-fs-nydus" ) +const ( + // Define the string key for DriverOptions in DeviceInfo struct + FsTypeOpt = "fstype" + BlockDriverOpt = "block-driver" +) + const ( // The OCI spec requires the major-minor number to be provided for a // device. We have chosen the below major numbers to represent @@ -97,7 +103,7 @@ var getSysDevPath = getSysDevPathImpl // DeviceInfo is an embedded type that contains device data common to all types of devices. type DeviceInfo struct { // DriverOptions is specific options for each device driver - // for example, for BlockDevice, we can set DriverOptions["blockDriver"]="virtio-blk" + // for example, for BlockDevice, we can set DriverOptions["block-driver"]="virtio-blk" DriverOptions map[string]string // Hostpath is device path on host diff --git a/src/runtime/virtcontainers/device/config/pmem.go b/src/runtime/virtcontainers/device/config/pmem.go index 81d1da9b57..44fd321873 100644 --- a/src/runtime/virtcontainers/device/config/pmem.go +++ b/src/runtime/virtcontainers/device/config/pmem.go @@ -81,8 +81,8 @@ func PmemDeviceInfo(source, destination string) (*DeviceInfo, error) { fstype = "ext4" } - pmemLog.WithField("fstype", fstype).Debug("filesystem for mount point") - device.DriverOptions["fstype"] = fstype + pmemLog.WithField(FsTypeOpt, fstype).Debug("filesystem for mount point") + device.DriverOptions[FsTypeOpt] = fstype return device, nil } diff --git a/src/runtime/virtcontainers/device/drivers/block.go b/src/runtime/virtcontainers/device/drivers/block.go index ce7eaceda1..faaadf32c9 100644 --- a/src/runtime/virtcontainers/device/drivers/block.go +++ b/src/runtime/virtcontainers/device/drivers/block.go @@ -70,13 +70,13 @@ func (device *BlockDevice) Attach(ctx context.Context, devReceiver api.DeviceRec ReadOnly: device.DeviceInfo.ReadOnly, } - if fs, ok := device.DeviceInfo.DriverOptions["fstype"]; ok { + if fs, ok := device.DeviceInfo.DriverOptions[config.FsTypeOpt]; ok { drive.Format = fs } customOptions := device.DeviceInfo.DriverOptions if customOptions == nil || - customOptions["block-driver"] == "virtio-scsi" { + customOptions[config.BlockDriverOpt] == config.VirtioSCSI { // User has not chosen a specific block device type // Default to SCSI scsiAddr, err := utils.GetSCSIAddress(index) @@ -85,15 +85,15 @@ func (device *BlockDevice) Attach(ctx context.Context, devReceiver api.DeviceRec } drive.SCSIAddr = scsiAddr - } else if customOptions["block-driver"] != "nvdimm" { + } else if customOptions[config.BlockDriverOpt] != config.Nvdimm { var globalIdx int - switch customOptions["block-driver"] { - case "virtio-blk": + switch customOptions[config.BlockDriverOpt] { + case config.VirtioBlock: globalIdx = index - case "virtio-blk-ccw": + case config.VirtioBlockCCW: globalIdx = index - case "virtio-mmio": + case config.VirtioMmio: //With firecracker the rootfs for the VM itself //sits at /dev/vda and consumes the first index. //Longer term block based VM rootfs should be added @@ -111,7 +111,7 @@ func (device *BlockDevice) Attach(ctx context.Context, devReceiver api.DeviceRec drive.VirtPath = filepath.Join("/dev", driveName) } - deviceLogger().WithField("device", device.DeviceInfo.HostPath).WithField("VirtPath", drive.VirtPath).Infof("Attaching %s device", customOptions["block-driver"]) + deviceLogger().WithField("device", device.DeviceInfo.HostPath).WithField("VirtPath", drive.VirtPath).Infof("Attaching %s device", customOptions[config.BlockDriverOpt]) device.BlockDrive = drive if err = devReceiver.HotplugAddDevice(ctx, device, config.DeviceBlock); err != nil { return err diff --git a/src/runtime/virtcontainers/device/drivers/vhost_user_blk.go b/src/runtime/virtcontainers/device/drivers/vhost_user_blk.go index 39dd2bd239..4a495c9d51 100644 --- a/src/runtime/virtcontainers/device/drivers/vhost_user_blk.go +++ b/src/runtime/virtcontainers/device/drivers/vhost_user_blk.go @@ -100,14 +100,14 @@ func isVirtioBlkBlockDriver(customOptions map[string]string) bool { if customOptions == nil { // User has not chosen a specific block device type // Default to SCSI - blockDriverOption = "virtio-scsi" + blockDriverOption = config.VirtioSCSI } else { - blockDriverOption = customOptions["block-driver"] + blockDriverOption = customOptions[config.BlockDriverOpt] } - if blockDriverOption == "virtio-blk" || - blockDriverOption == "virtio-blk-ccw" || - blockDriverOption == "virtio-mmio" { + if blockDriverOption == config.VirtioBlock || + blockDriverOption == config.VirtioBlockCCW || + blockDriverOption == config.VirtioMmio { return true } diff --git a/src/runtime/virtcontainers/device/manager/manager.go b/src/runtime/virtcontainers/device/manager/manager.go index 195187ba82..9a5bba2604 100644 --- a/src/runtime/virtcontainers/device/manager/manager.go +++ b/src/runtime/virtcontainers/device/manager/manager.go @@ -21,19 +21,6 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) -const ( - // VirtioMmio indicates block driver is virtio-mmio based - VirtioMmio string = "virtio-mmio" - // VirtioBlock indicates block driver is virtio-blk based - VirtioBlock string = "virtio-blk" - // VirtioBlockCCW indicates block driver is virtio-blk-ccw based - VirtioBlockCCW string = "virtio-blk-ccw" - // VirtioSCSI indicates block driver is virtio-scsi based - VirtioSCSI string = "virtio-scsi" - // Nvdimm indicates block driver is nvdimm based - Nvdimm string = "nvdimm" -) - var ( // ErrIDExhausted represents that devices are too many // and no more IDs can be generated @@ -69,16 +56,16 @@ func NewDeviceManager(blockDriver string, vhostUserStoreEnabled bool, vhostUserS vhostUserStorePath: vhostUserStorePath, devices: make(map[string]api.Device), } - if blockDriver == VirtioMmio { - dm.blockDriver = VirtioMmio - } else if blockDriver == VirtioBlock { - dm.blockDriver = VirtioBlock - } else if blockDriver == Nvdimm { - dm.blockDriver = Nvdimm - } else if blockDriver == VirtioBlockCCW { - dm.blockDriver = VirtioBlockCCW + if blockDriver == config.VirtioMmio { + dm.blockDriver = config.VirtioMmio + } else if blockDriver == config.VirtioBlock { + dm.blockDriver = config.VirtioBlock + } else if blockDriver == config.Nvdimm { + dm.blockDriver = config.Nvdimm + } else if blockDriver == config.VirtioBlockCCW { + dm.blockDriver = config.VirtioBlockCCW } else { - dm.blockDriver = VirtioSCSI + dm.blockDriver = config.VirtioSCSI } drivers.AllPCIeDevs = make(map[string]bool) @@ -132,13 +119,13 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device if devInfo.DriverOptions == nil { devInfo.DriverOptions = make(map[string]string) } - devInfo.DriverOptions["block-driver"] = dm.blockDriver + devInfo.DriverOptions[config.BlockDriverOpt] = dm.blockDriver return drivers.NewVhostUserBlkDevice(&devInfo), nil } else if isBlock(devInfo) { if devInfo.DriverOptions == nil { devInfo.DriverOptions = make(map[string]string) } - devInfo.DriverOptions["block-driver"] = dm.blockDriver + devInfo.DriverOptions[config.BlockDriverOpt] = dm.blockDriver return drivers.NewBlockDevice(&devInfo), nil } else { deviceLogger().WithField("device", devInfo.HostPath).Info("Device has not been passed to the container") diff --git a/src/runtime/virtcontainers/device/manager/manager_linux_test.go b/src/runtime/virtcontainers/device/manager/manager_linux_test.go index 78773fc5c2..b47a38c0ad 100644 --- a/src/runtime/virtcontainers/device/manager/manager_linux_test.go +++ b/src/runtime/virtcontainers/device/manager/manager_linux_test.go @@ -31,7 +31,7 @@ func TestAttachVhostUserBlkDevice(t *testing.T) { tmpDir, err := os.MkdirTemp("", "") dm := &deviceManager{ - blockDriver: VirtioBlock, + blockDriver: config.VirtioBlock, devices: make(map[string]api.Device), vhostUserStoreEnabled: true, vhostUserStorePath: tmpDir, diff --git a/src/runtime/virtcontainers/device/manager/manager_test.go b/src/runtime/virtcontainers/device/manager/manager_test.go index f0d7ef974d..8e1b8ec4bf 100644 --- a/src/runtime/virtcontainers/device/manager/manager_test.go +++ b/src/runtime/virtcontainers/device/manager/manager_test.go @@ -26,7 +26,7 @@ const dirMode = os.FileMode(0750) | os.ModeDir func TestNewDevice(t *testing.T) { dm := &deviceManager{ - blockDriver: VirtioBlock, + blockDriver: config.VirtioBlock, devices: make(map[string]api.Device), } savedSysDevPrefix := config.SysDevPrefix @@ -96,7 +96,7 @@ func TestNewDevice(t *testing.T) { func TestAttachVFIODevice(t *testing.T) { dm := &deviceManager{ - blockDriver: VirtioBlock, + blockDriver: config.VirtioBlock, devices: make(map[string]api.Device), } tmpDir, err := os.MkdirTemp("", "") @@ -155,7 +155,7 @@ func TestAttachVFIODevice(t *testing.T) { func TestAttachGenericDevice(t *testing.T) { dm := &deviceManager{ - blockDriver: VirtioBlock, + blockDriver: config.VirtioBlock, devices: make(map[string]api.Device), } path := "/dev/tty2" @@ -180,7 +180,7 @@ func TestAttachGenericDevice(t *testing.T) { func TestAttachBlockDevice(t *testing.T) { dm := &deviceManager{ - blockDriver: VirtioBlock, + blockDriver: config.VirtioBlock, devices: make(map[string]api.Device), } path := "/dev/hda" @@ -203,7 +203,7 @@ func TestAttachBlockDevice(t *testing.T) { assert.Nil(t, err) // test virtio SCSI driver - dm.blockDriver = VirtioSCSI + dm.blockDriver = config.VirtioSCSI device, err = dm.NewDevice(deviceInfo) assert.Nil(t, err) err = device.Attach(context.Background(), devReceiver) @@ -214,7 +214,7 @@ func TestAttachBlockDevice(t *testing.T) { } func TestAttachDetachDevice(t *testing.T) { - dm := NewDeviceManager(VirtioSCSI, false, "", nil) + dm := NewDeviceManager(config.VirtioSCSI, false, "", nil) path := "/dev/hda" deviceInfo := config.DeviceInfo{ diff --git a/src/runtime/virtcontainers/documentation/api/1.0/api.md b/src/runtime/virtcontainers/documentation/api/1.0/api.md index a2e1a55ff1..6455e19d0b 100644 --- a/src/runtime/virtcontainers/documentation/api/1.0/api.md +++ b/src/runtime/virtcontainers/documentation/api/1.0/api.md @@ -547,7 +547,7 @@ type DeviceInfo struct { ID string // DriverOptions is specific options for each device driver - // for example, for BlockDevice, we can set DriverOptions["blockDriver"]="virtio-blk" + // for example, for BlockDevice, we can set DriverOptions["block-driver"]="virtio-blk" DriverOptions map[string]string } ``` @@ -835,7 +835,7 @@ type DeviceInfo struct { ID string // DriverOptions is specific options for each device driver - // for example, for BlockDevice, we can set DriverOptions["blockDriver"]="virtio-blk" + // for example, for BlockDevice, we can set DriverOptions["block-driver"]="virtio-blk" DriverOptions map[string]string } ``` diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index f494626c62..bcca754ba6 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -390,10 +390,10 @@ func TestHandleBlockVolume(t *testing.T) { mounts = append(mounts, vMount, bMount, dMount) tmpDir := "/vhost/user/dir" - dm := manager.NewDeviceManager(manager.VirtioBlock, true, tmpDir, devices) + dm := manager.NewDeviceManager(config.VirtioBlock, true, tmpDir, devices) sConfig := SandboxConfig{} - sConfig.HypervisorConfig.BlockDeviceDriver = manager.VirtioBlock + sConfig.HypervisorConfig.BlockDeviceDriver = config.VirtioBlock sandbox := Sandbox{ id: "100", containers: containers, diff --git a/src/runtime/virtcontainers/persist/api/device.go b/src/runtime/virtcontainers/persist/api/device.go index acb05fc674..dd61efbfcf 100644 --- a/src/runtime/virtcontainers/persist/api/device.go +++ b/src/runtime/virtcontainers/persist/api/device.go @@ -86,7 +86,7 @@ type VhostUserDeviceAttrs struct { // Refs: virtcontainers/device/drivers/generic.go:GenericDevice type DeviceState struct { // DriverOptions is specific options for each device driver - // for example, for BlockDevice, we can set DriverOptions["blockDriver"]="virtio-blk" + // for example, for BlockDevice, we can set DriverOptions["block-driver"]="virtio-blk" DriverOptions map[string]string // VhostUserDeviceAttrs is specific for vhost-user device driver diff --git a/src/runtime/virtcontainers/persist_test.go b/src/runtime/virtcontainers/persist_test.go index 228638a960..b6c096a5d9 100644 --- a/src/runtime/virtcontainers/persist_test.go +++ b/src/runtime/virtcontainers/persist_test.go @@ -10,11 +10,11 @@ import ( "os" "testing" - "github.com/stretchr/testify/assert" - + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" + "github.com/stretchr/testify/assert" ) func TestSandboxRestore(t *testing.T) { @@ -32,7 +32,7 @@ func TestSandboxRestore(t *testing.T) { sandbox := Sandbox{ id: "test-exp", containers: container, - devManager: manager.NewDeviceManager(manager.VirtioSCSI, false, "", nil), + devManager: manager.NewDeviceManager(config.VirtioSCSI, false, "", nil), hypervisor: &mockHypervisor{}, network: network, ctx: context.Background(), diff --git a/src/runtime/virtcontainers/sandbox_test.go b/src/runtime/virtcontainers/sandbox_test.go index 344551e3b9..b1556ec156 100644 --- a/src/runtime/virtcontainers/sandbox_test.go +++ b/src/runtime/virtcontainers/sandbox_test.go @@ -548,7 +548,7 @@ func TestSandboxAttachDevicesVFIO(t *testing.T) { config.SysIOMMUPath = savedIOMMUPath }() - dm := manager.NewDeviceManager(manager.VirtioSCSI, false, "", nil) + dm := manager.NewDeviceManager(config.VirtioSCSI, false, "", nil) path := filepath.Join(vfioPath, testFDIOGroup) deviceInfo := config.DeviceInfo{ HostPath: path, @@ -599,7 +599,7 @@ func TestSandboxAttachDevicesVhostUserBlk(t *testing.T) { tmpDir, err := os.MkdirTemp("", "") assert.Nil(t, err) os.RemoveAll(tmpDir) - dm := manager.NewDeviceManager(manager.VirtioSCSI, true, tmpDir, nil) + dm := manager.NewDeviceManager(config.VirtioSCSI, true, tmpDir, nil) vhostUserDevNodePath := filepath.Join(tmpDir, "/block/devices/") vhostUserSockPath := filepath.Join(tmpDir, "/block/sockets/") From aa5ae6b17c688a3fa22c28d1ec3a71a6cabb9c57 Mon Sep 17 00:00:00 2001 From: Feng Wang Date: Fri, 11 Mar 2022 10:44:26 -0800 Subject: [PATCH 23/26] runtime: Properly handle ESRCH error when signaling container Currently kata shim v2 doesn't translate ESRCH signal, causing container fail to stop and shim leak. Fixes: #3874 Signed-off-by: Feng Wang --- src/runtime/virtcontainers/container.go | 14 +++++++++++++- src/runtime/virtcontainers/utils/utils.go | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index b95e72ddc4..d82124c1d6 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -12,6 +12,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "syscall" "time" @@ -1060,7 +1061,18 @@ func (c *Container) signalProcess(ctx context.Context, processID string, signal return fmt.Errorf("Container not ready, running or paused, impossible to signal the container") } - return c.sandbox.agent.signalProcess(ctx, c, processID, signal, all) + // kill(2) method can return ESRCH in certain cases, which is not handled by containerd cri server in container_stop.go. + // CRIO server also doesn't handle ESRCH. So kata runtime will swallow it here. + var err error + if err = c.sandbox.agent.signalProcess(ctx, c, processID, signal, all); err != nil && + strings.Contains(err.Error(), "ESRCH: No such process") { + c.Logger().WithFields(logrus.Fields{ + "container": c.id, + "process-id": processID, + }).Warn("signal encounters ESRCH, process already finished") + return nil + } + return err } func (c *Container) winsizeProcess(ctx context.Context, processID string, height, width uint32) error { diff --git a/src/runtime/virtcontainers/utils/utils.go b/src/runtime/virtcontainers/utils/utils.go index f03a5aa259..88c29cec5a 100644 --- a/src/runtime/virtcontainers/utils/utils.go +++ b/src/runtime/virtcontainers/utils/utils.go @@ -321,6 +321,7 @@ func WaitLocalProcess(pid int, timeoutSecs uint, initialSignal syscall.Signal, l if initialSignal != syscall.Signal(0) { if err = syscall.Kill(pid, initialSignal); err != nil { if err == syscall.ESRCH { + logger.WithField("pid", pid).Warnf("kill encounters ESRCH, process already finished") return nil } From 62351637da28eebdc3a68f772fb51c92d8125f21 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Tue, 15 Mar 2022 16:11:43 +0000 Subject: [PATCH 24/26] action: Update link for format patch documentation This PR updates the link for the format patch documentation for the commit message check. Fixes #3900 Signed-off-by: Gabriela Cervantes --- .github/workflows/commit-message-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/commit-message-check.yaml b/.github/workflows/commit-message-check.yaml index b76484b976..2062e51b9c 100644 --- a/.github/workflows/commit-message-check.yaml +++ b/.github/workflows/commit-message-check.yaml @@ -10,7 +10,7 @@ env: error_msg: |+ See the document below for help on formatting commits for the project. - https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md#patch-format + https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md#patch-format jobs: commit-message-check: From 18d4d7fb1da93ea42029219e40709dbd00725f89 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Tue, 15 Mar 2022 15:21:24 -0600 Subject: [PATCH 25/26] tools: update QEMU to 6.2 bring Intel SGX support Changes tha may impact in Kata Containers Arm: The 'virt' machine now supports an emulated ITS The 'virt' machine now supports more than 123 CPUs in TCG emulation mode The pl031 real-time clock device now supports sending RTC_CHANGE QMP events PowerPC: Improved POWER10 support for the 'powernv' machine Initial support for POWER10 DD2.0 CPU added Added support for FORM2 PAPR NUMA descriptions in the "pseries" machine type s390x: Improved storage key emulation (e.g. fixed address handling, lazy storage key enablement for TCG, ...) New gen16 CPU features are now enabled automatically in the latest machine type KVM: Support for SGX in the virtual machine, using the /dev/sgx_vepc device on the host and the "memory-backend-epc" backend in QEMU. New "hv-apicv" CPU property (aliased to "hv-avic") sets the HV_DEPRECATING_AEOI_RECOMMENDED bit in CPUID[0x40000004].EAX. virtio-mem: QEMU now fully supports guest memory dumps with virtio-mem. QEMU now cleanly supports precopy migration, postcopy migration and background snapshots with virtio-mem. fixes #3902 Signed-off-by: Julio Montes --- ...ve-SPR_DSISR-setting-to-powerpc_excp.patch | 81 +++++++++++++++++++ .../patches/tag_patches/v6.2.0/no_patches.txt | 0 .../packaging/scripts/configure-hypervisor.sh | 2 - versions.yaml | 4 +- 4 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 tools/packaging/qemu/patches/6.2.x/Revert-target-ppc-Move-SPR_DSISR-setting-to-powerpc_excp.patch create mode 100644 tools/packaging/qemu/patches/tag_patches/v6.2.0/no_patches.txt diff --git a/tools/packaging/qemu/patches/6.2.x/Revert-target-ppc-Move-SPR_DSISR-setting-to-powerpc_excp.patch b/tools/packaging/qemu/patches/6.2.x/Revert-target-ppc-Move-SPR_DSISR-setting-to-powerpc_excp.patch new file mode 100644 index 0000000000..caccdb49ad --- /dev/null +++ b/tools/packaging/qemu/patches/6.2.x/Revert-target-ppc-Move-SPR_DSISR-setting-to-powerpc_excp.patch @@ -0,0 +1,81 @@ +From 29c4a3363bf287bb9a7b0342b1bc2dba3661c96c Mon Sep 17 00:00:00 2001 +From: Fabiano Rosas +Date: Fri, 17 Dec 2021 17:57:18 +0100 +Subject: [PATCH] Revert "target/ppc: Move SPR_DSISR setting to powerpc_excp" +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This reverts commit 336e91f85332dda0ede4c1d15b87a19a0fb898a2. + +It breaks the --disable-tcg build: + + ../target/ppc/excp_helper.c:463:29: error: implicit declaration of + function ‘cpu_ldl_code’ [-Werror=implicit-function-declaration] + +We should not have TCG code in powerpc_excp because some kvm-only +routines use it indirectly to dispatch interrupts. See +kvm_handle_debug, spapr_mce_req_event and +spapr_do_system_reset_on_cpu. + +We can re-introduce the change once we have split the interrupt +injection code between KVM and TCG. + +Signed-off-by: Fabiano Rosas +Message-Id: <20211209173323.2166642-1-farosas@linux.ibm.com> +Signed-off-by: Cédric Le Goater +--- + target/ppc/excp_helper.c | 21 ++++++++++++--------- + 1 file changed, 12 insertions(+), 9 deletions(-) + +diff --git a/target/ppc/excp_helper.c b/target/ppc/excp_helper.c +index feb3fd42e2..6ba0840e99 100644 +--- a/target/ppc/excp_helper.c ++++ b/target/ppc/excp_helper.c +@@ -464,15 +464,13 @@ static inline void powerpc_excp(PowerPCCPU *cpu, int excp_model, int excp) + break; + } + case POWERPC_EXCP_ALIGN: /* Alignment exception */ ++ /* Get rS/rD and rA from faulting opcode */ + /* +- * Get rS/rD and rA from faulting opcode. +- * Note: We will only invoke ALIGN for atomic operations, +- * so all instructions are X-form. ++ * Note: the opcode fields will not be set properly for a ++ * direct store load/store, but nobody cares as nobody ++ * actually uses direct store segments. + */ +- { +- uint32_t insn = cpu_ldl_code(env, env->nip); +- env->spr[SPR_DSISR] |= (insn & 0x03FF0000) >> 16; +- } ++ env->spr[SPR_DSISR] |= (env->error_code & 0x03FF0000) >> 16; + break; + case POWERPC_EXCP_PROGRAM: /* Program exception */ + switch (env->error_code & ~0xF) { +@@ -1441,6 +1439,11 @@ void ppc_cpu_do_unaligned_access(CPUState *cs, vaddr vaddr, + int mmu_idx, uintptr_t retaddr) + { + CPUPPCState *env = cs->env_ptr; ++ uint32_t insn; ++ ++ /* Restore state and reload the insn we executed, for filling in DSISR. */ ++ cpu_restore_state(cs, retaddr, true); ++ insn = cpu_ldl_code(env, env->nip); + + switch (env->mmu_model) { + case POWERPC_MMU_SOFT_4xx: +@@ -1456,8 +1459,8 @@ void ppc_cpu_do_unaligned_access(CPUState *cs, vaddr vaddr, + } + + cs->exception_index = POWERPC_EXCP_ALIGN; +- env->error_code = 0; +- cpu_loop_exit_restore(cs, retaddr); ++ env->error_code = insn & 0x03FF0000; ++ cpu_loop_exit(cs); + } + #endif /* CONFIG_TCG */ + #endif /* !CONFIG_USER_ONLY */ +-- +GitLab + diff --git a/tools/packaging/qemu/patches/tag_patches/v6.2.0/no_patches.txt b/tools/packaging/qemu/patches/tag_patches/v6.2.0/no_patches.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/packaging/scripts/configure-hypervisor.sh b/tools/packaging/scripts/configure-hypervisor.sh index 1d3689927e..1e3e2f6088 100755 --- a/tools/packaging/scripts/configure-hypervisor.sh +++ b/tools/packaging/scripts/configure-hypervisor.sh @@ -250,7 +250,6 @@ generate_qemu_options() { qemu_options+=(size:--disable-auth-pam) # Disable unused filesystem support - [ "$arch" == x86_64 ] && qemu_options+=(size:--disable-fdt) qemu_options+=(size:--disable-glusterfs) qemu_options+=(size:--disable-libiscsi) qemu_options+=(size:--disable-libnfs) @@ -303,7 +302,6 @@ generate_qemu_options() { ;; esac qemu_options+=(size:--disable-qom-cast-debug) - qemu_options+=(size:--disable-tcmalloc) # Disable libudev since it is only needed for qemu-pr-helper and USB, # none of which are used with Kata diff --git a/versions.yaml b/versions.yaml index 3384287432..d1ad5c690b 100644 --- a/versions.yaml +++ b/versions.yaml @@ -88,8 +88,8 @@ assets: qemu: description: "VMM that uses KVM" url: "https://github.com/qemu/qemu" - version: "v6.1.0" - tag: "v6.1.0" + version: "v6.2.0" + tag: "v6.2.0" # Do not include any non-full release versions # Break the line *without CR or space being appended*, to appease # yamllint, and note the deliberate ' ' at the end of the expression. From 24b29310b289fce67e434ab83e26d2d7c045b364 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Fri, 18 Mar 2022 09:03:51 -0600 Subject: [PATCH 26/26] doc: update Intel SGX use cases document Installation section is not longer needed because of the latest default kata kernel supports Intel SGX. Include QEMU to the list of supported hypervisors. fixes #3911 Signed-off-by: Julio Montes --- docs/use-cases/using-Intel-SGX-and-kata.md | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/docs/use-cases/using-Intel-SGX-and-kata.md b/docs/use-cases/using-Intel-SGX-and-kata.md index 5427331ed9..f45e3ed5bc 100644 --- a/docs/use-cases/using-Intel-SGX-and-kata.md +++ b/docs/use-cases/using-Intel-SGX-and-kata.md @@ -21,20 +21,7 @@ CONFIG_X86_SGX_KVM=y * [Intel SGX Kubernetes device plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/sgx_plugin#deploying-with-pre-built-images) > Note: Kata Containers supports creating VM sandboxes with Intel® SGX enabled -> using [cloud-hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor/) VMM only. QEMU support is waiting to get the -> Intel SGX enabled QEMU upstream release. - -## Installation - -### Kata Containers Guest Kernel - -Follow the instructions to [setup](../../tools/packaging/kernel/README.md#setup-kernel-source-code) and [build](../../tools/packaging/kernel/README.md#build-the-kernel) the experimental guest kernel. Then, install as: - -```sh -$ sudo cp kata-linux-experimental-*/vmlinux /opt/kata/share/kata-containers/vmlinux.sgx -$ sudo sed -i 's|vmlinux.container|vmlinux.sgx|g' \ - /opt/kata/share/defaults/kata-containers/configuration-clh.toml -``` +> using [cloud-hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor/) and [QEMU](https://www.qemu.org/) VMMs only. ### Kata Containers Configuration @@ -48,6 +35,8 @@ to the `sandbox` are: `["io.katacontainers.*", "sgx.intel.com/epc"]`. With the following sample job deployed using `kubectl apply -f`: +> Note: Change the `runtimeClassName` option accordingly, only `kata-clh` and `kata-qemu` support Intel® SGX. + ```yaml apiVersion: batch/v1 kind: Job