From ba858e8cd99fbc9e1657a9bd3b3d5926691c2359 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 24 Feb 2022 12:41:33 +1100 Subject: [PATCH 01/29] agent: Don't attempt to create directories for hugepage configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit allocate_hugepages() constructs the path for the sysfs directory containing hugepage configuration, then attempts to create this directory if it does not exist. This doesn't make sense: sysfs is a view into kernel configuration, if the kernel has support for the hugepage size, the directory will already be there, if it doesn't, trying to create it won't help. For the same reason, attempting to create the "nr_hugepages" file itself is pointless, so there's no reason to call OpenOptions::create(true). Signed-off-by: David Gibson (cherry picked from commit 608e003abcf7ce54c12512d79b5196b74c1883db) Signed-off-by: Fabiano Fidêncio --- src/agent/src/mount.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index fbead0953a..e788e18dff 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -321,20 +321,16 @@ fn allocate_hugepages(logger: &Logger, options: &[String]) -> Result<()> { // sysfs entry is always of the form hugepages-${pagesize}kB // Ref: https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt - let path = Path::new(SYS_FS_HUGEPAGES_PREFIX).join(format!("hugepages-{}kB", pagesize / 1024)); - - if !path.exists() { - fs::create_dir_all(&path).context("create hugepages-size directory")?; - } + let path = Path::new(SYS_FS_HUGEPAGES_PREFIX) + .join(format!("hugepages-{}kB", pagesize / 1024)) + .join("nr_hugepages"); // write numpages to nr_hugepages file. - let path = path.join("nr_hugepages"); let numpages = format!("{}", size / pagesize); info!(logger, "write {} pages to {:?}", &numpages, &path); let mut file = OpenOptions::new() .write(true) - .create(true) .open(&path) .context(format!("open nr_hugepages directory {:?}", &path))?; From c9e1f7278505fb20b66b3152be664cc6634a6aaa Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 24 Feb 2022 12:53:49 +1100 Subject: [PATCH 02/29] agent: Verify that we allocated as many hugepages as we need MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit allocate_hugepages() writes to the kernel sysfs file to allocate hugepages in the Kata VM. However, even if the write succeeds, it's not certain that the kernel will actually be able to allocate as many hugepages as we requested. This patch reads back the file after writing it to check if we were able to allocate all the required hugepages. fixes #3816 Signed-off-by: David Gibson (cherry picked from commit 42e35505b0fb1f207ca041a4d22de05d41106be8) Signed-off-by: Fabiano Fidêncio --- src/agent/src/mount.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index e788e18dff..f4b50af6ec 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -337,6 +337,23 @@ fn allocate_hugepages(logger: &Logger, options: &[String]) -> Result<()> { file.write_all(numpages.as_bytes()) .context(format!("write nr_hugepages failed: {:?}", &path))?; + // Even if the write succeeds, the kernel isn't guaranteed to be + // able to allocate all the pages we requested. Verify that it + // did. + let verify = fs::read_to_string(&path).context(format!("reading {:?}", &path))?; + let allocated = verify + .trim_end() + .parse::() + .map_err(|_| anyhow!("Unexpected text {:?} in {:?}", &verify, &path))?; + if allocated != size / pagesize { + return Err(anyhow!( + "Only allocated {} of {} hugepages of size {}", + allocated, + numpages, + pagesize + )); + } + Ok(()) } From aedfef29a3bb2f7064beda4872d42438dd1fbf96 Mon Sep 17 00:00:00 2001 From: Miao Xia Date: Thu, 17 Feb 2022 11:15:36 +0800 Subject: [PATCH 03/29] runtime/virtcontainers: Pass the hugepages resources to agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hugepages resources claimed by containers should be limited by cgroup in the guest OS. Fixes: #3695 Signed-off-by: Miao Xia (cherry picked from commit a2f5c1768e49883560125713b157be7dcf41210e) Signed-off-by: Fabiano Fidêncio --- src/runtime/virtcontainers/kata_agent.go | 1 - src/runtime/virtcontainers/kata_agent_test.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index c2a8c901f4..834a12b55e 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -910,7 +910,6 @@ func (k *kataAgent) constrainGRPCSpec(grpcSpec *grpc.Spec, passSeccomp bool, str grpcSpec.Linux.Resources.Devices = nil grpcSpec.Linux.Resources.Pids = nil grpcSpec.Linux.Resources.BlockIO = nil - grpcSpec.Linux.Resources.HugepageLimits = nil grpcSpec.Linux.Resources.Network = nil if grpcSpec.Linux.Resources.CPU != nil { grpcSpec.Linux.Resources.CPU.Cpus = "" diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index f494626c62..1dbfe45698 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -609,7 +609,7 @@ func TestConstrainGRPCSpec(t *testing.T) { assert.NotNil(g.Linux.Resources.Memory) assert.Nil(g.Linux.Resources.Pids) assert.Nil(g.Linux.Resources.BlockIO) - assert.Nil(g.Linux.Resources.HugepageLimits) + assert.Len(g.Linux.Resources.HugepageLimits, 0) assert.Nil(g.Linux.Resources.Network) assert.NotNil(g.Linux.Resources.CPU) assert.Equal(g.Process.SelinuxLabel, "") From 3049b7760ac731ec00ea6c020e1d253b07041250 Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Thu, 31 Mar 2022 13:19:01 +0200 Subject: [PATCH 04/29] versions: Bump firecracker to v0.23.4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This release changes Docker images repository from DockerHub to Amazon ECR. This resolves the `You have reached your pull rate limit` error when building the firecracker tarball. Fixes #4001 Signed-off-by: Greg Kurz (cherry picked from commit 0d5f80b803987dab1e5fa12b5ca5fc9954dd9b89) Signed-off-by: Fabiano Fidêncio --- versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.yaml b/versions.yaml index d033f80036..6fab525a35 100644 --- a/versions.yaml +++ b/versions.yaml @@ -83,7 +83,7 @@ assets: uscan-url: >- https://github.com/firecracker-microvm/firecracker/tags .*/v?(\d\S+)\.tar\.gz - version: "v0.23.1" + version: "v0.23.4" qemu: description: "VMM that uses KVM" From 8f67f9e384e2fd99fa5b5949a6ee4fa0de477488 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Wed, 2 Mar 2022 16:24:42 +1100 Subject: [PATCH 05/29] tools/packaging/kata-deploy/local-build: Add build to gitignore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This directory consists entirely of files built during a make kata-tarball, so it should not be committed to the tree. A symbolic link to this directory might be created during 'make tarball', ignore it as well. Signed-off-by: David Gibson [greg: - rearranged the subject to make the subsystem checker happy - also ignore the symbolic link created by `kata-deploy-binaries-in-docker.sh`] Signed-off-by: Greg Kurz (cherry picked from commit bad859d2f8d51aa72e563c30bac638c7415a5329) Signed-off-by: Fabiano Fidêncio --- .gitignore | 1 + tools/packaging/kata-deploy/local-build/.gitignore | 1 + 2 files changed, 2 insertions(+) create mode 100644 tools/packaging/kata-deploy/local-build/.gitignore diff --git a/.gitignore b/.gitignore index 529bab04a7..ce97c7e983 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ src/agent/src/version.rs src/agent/kata-agent.service src/agent/protocols/src/*.rs !src/agent/protocols/src/lib.rs +build diff --git a/tools/packaging/kata-deploy/local-build/.gitignore b/tools/packaging/kata-deploy/local-build/.gitignore new file mode 100644 index 0000000000..567609b123 --- /dev/null +++ b/tools/packaging/kata-deploy/local-build/.gitignore @@ -0,0 +1 @@ +build/ From 8fa64e011d66f925bdd4e7f7a716889979e18e09 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Wed, 2 Mar 2022 16:26:22 +1100 Subject: [PATCH 06/29] packaging: Eliminate TTY_OPT and NO_TTY variables in kata-deploy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NO_TTY configured whether to add the -t option to docker run. It makes no sense for the caller to configure this, since whether you need it depends on the commands you're running. Since the point here is to run non-interactive build scripts, we don't need -t, or -i either. Signed-off-by: David Gibson Signed-off-by: Greg Kurz (cherry picked from commit 1ed7da8fc75426e71cf6d68648ab6ab4576f0a54) Signed-off-by: Fabiano Fidêncio --- tools/packaging/kata-deploy/local-build/Makefile | 2 +- .../local-build/kata-deploy-binaries-in-docker.sh | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile index 68e45d447e..f04d9b6581 100644 --- a/tools/packaging/kata-deploy/local-build/Makefile +++ b/tools/packaging/kata-deploy/local-build/Makefile @@ -16,7 +16,7 @@ endef kata-tarball: | all-parallel merge-builds all-parallel: - ${MAKE} -f $(MK_PATH) all -j$$(( $$(nproc) - 1 )) NO_TTY="true" V= + ${MAKE} -f $(MK_PATH) all -j$$(( $$(nproc) - 1 )) V= all: cloud-hypervisor-tarball \ firecracker-tarball \ diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh index 68fba1b5f2..c1fc1538a3 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh @@ -16,10 +16,6 @@ kata_deploy_create="${script_dir}/kata-deploy-binaries.sh" uid=$(id -u ${USER}) gid=$(id -g ${USER}) -TTY_OPT="-i" -NO_TTY="${NO_TTY:-false}" -[ -t 1 ] && [ "${NO_TTY}" == "false" ] && TTY_OPT="-it" - if [ "${script_dir}" != "${PWD}" ]; then ln -sf "${script_dir}/build" "${PWD}/build" fi @@ -34,7 +30,7 @@ docker build -q -t build-kata-deploy \ --build-arg GID=${gid} \ "${script_dir}/dockerbuild/" -docker run ${TTY_OPT} \ +docker run \ -v /var/run/docker.sock:/var/run/docker.sock \ --user ${uid}:${gid} \ --env USER=${USER} -v "${kata_dir}:${kata_dir}" \ From 1e6223161019b9491f856328d950626aa4446445 Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Fri, 25 Mar 2022 10:30:52 +0100 Subject: [PATCH 07/29] tools/packaging/kata-deploy: Copy install_yq.sh in a dedicated script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 'make kata-tarball' sometimes fails early with: cp: cannot create regular file '[...]/tools/packaging/kata-deploy/local-build/dockerbuild/install_yq.sh': File exists This happens because all assets are built in parallel using the same `kata-deploy-binaries-in-docker.sh` script, and thus all try to copy the `install_yq.sh` script to the same location with the `cp` command. This is a well known race condition that cannot be avoided without serialization of `cp` invocations. Move the copying of `install_yq.sh` to a separate script and ensure it is called *before* parallel builds. Make the presence of the copy a prerequisite for each sub-build so that they still can be triggered individually. Update the GH release workflow to also call this script before calling `kata-deploy-binaries-in-docker.sh`. Fixes #3756 Signed-off-by: Greg Kurz (cherry picked from commit 154c8b03d3763bfe100155105a7db7e889bf2a38) Signed-off-by: Fabiano Fidêncio --- .github/workflows/release.yaml | 1 + tools/packaging/kata-deploy/local-build/Makefile | 7 +++++-- .../kata-deploy-binaries-in-docker.sh | 4 ---- .../local-build/kata-deploy-copy-yq-installer.sh | 16 ++++++++++++++++ 4 files changed, 22 insertions(+), 6 deletions(-) create mode 100755 tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d29872e101..56cc895389 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -26,6 +26,7 @@ jobs: - name: Build ${{ matrix.asset }} run: | + ./tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh ./tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh --build="${KATA_ASSET}" build_dir=$(readlink -f build) # store-artifact does not work with symlink diff --git a/tools/packaging/kata-deploy/local-build/Makefile b/tools/packaging/kata-deploy/local-build/Makefile index f04d9b6581..ac3a77714a 100644 --- a/tools/packaging/kata-deploy/local-build/Makefile +++ b/tools/packaging/kata-deploy/local-build/Makefile @@ -15,7 +15,10 @@ endef kata-tarball: | all-parallel merge-builds -all-parallel: +$(MK_DIR)/dockerbuild/install_yq.sh: + $(MK_DIR)/kata-deploy-copy-yq-installer.sh + +all-parallel: $(MK_DIR)/dockerbuild/install_yq.sh ${MAKE} -f $(MK_PATH) all -j$$(( $$(nproc) - 1 )) V= all: cloud-hypervisor-tarball \ @@ -26,7 +29,7 @@ all: cloud-hypervisor-tarball \ rootfs-initrd-tarball \ shim-v2-tarball -%-tarball-build: +%-tarball-build: $(MK_DIR)/dockerbuild/install_yq.sh $(call BUILD,$*) cloud-hypervisor-tarball: diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh index c1fc1538a3..4035ff9cbd 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh @@ -20,10 +20,6 @@ if [ "${script_dir}" != "${PWD}" ]; then ln -sf "${script_dir}/build" "${PWD}/build" fi -install_yq_script_path="${script_dir}/../../../../ci/install_yq.sh" - -cp "${install_yq_script_path}" "${script_dir}/dockerbuild/install_yq.sh" - docker build -q -t build-kata-deploy \ --build-arg IMG_USER="${USER}" \ --build-arg UID=${uid} \ diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh new file mode 100755 index 0000000000..1271fd8826 --- /dev/null +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-copy-yq-installer.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018-2021 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +script_dir=$(dirname "$(readlink -f "$0")") +install_yq_script_path="${script_dir}/../../../../ci/install_yq.sh" + +cp "${install_yq_script_path}" "${script_dir}/dockerbuild/install_yq.sh" From c9c77511845c244ad42ffe085d6a975cea514dec Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Tue, 29 Mar 2022 17:32:29 +0200 Subject: [PATCH 08/29] tools/packaging: Fix usage of kata-deploy-binaries.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add missing documentation for -s . Signed-off-by: Greg Kurz (cherry picked from commit 0baebd2b374082a1cd6349053445eacdb51fbc73) Signed-off-by: Fabiano Fidêncio --- tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 2a215726b9..19c85c1108 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -64,6 +64,7 @@ version: The kata version that will be use to create the tarball options: -h|--help : Show this help +-s : Silent mode (produce output in case of failure only) --build= : all cloud-hypervisor From d1197ee8e5572f2d5e46d8fe4df8ea37bc16d677 Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Tue, 29 Mar 2022 15:10:38 +0200 Subject: [PATCH 09/29] tools/packaging: Fix error path in 'kata-deploy-binaries.sh -s' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `make kata-tarball` relies on `kata-deploy-binaries.sh -s` which silently ignores errors, and you may end up with an incomplete tarball without noticing it because `make`'s exit status is 0. `kata-deploy-binaries.sh` does set the `errexit` option and all the code in the script seems to assume that since it doesn't do error checking. Unfortunately, bash automatically disables `errexit` when calling a function from a conditional pipeline, like done in the `-s` case: if [ "${silent}" == true ]; then if ! handle_build "${t}" &>"$log_file"; then ^^^^^^ this disables `errexit` and `handle_build` ends with a `tar tvf` that always succeeds. Adding error checking all over the place isn't really an option as it would seriously obfuscate the code. Drop the conditional pipeline instead and print the final error message from a `trap` handler on the special ERR signal. This requires the `errtrace` option as `trap`s aren't propagated to functions by default. Since all outputs of `handle_build` are redirected to the build log file, some file descriptor duplication magic is needed for the handler to be able to write to the orignal stdout and stderr. Fixes #3757 Signed-off-by: Greg Kurz (cherry picked from commit a779e19beef0fdb50b905b25aead0294d4f0cb19) Signed-off-by: Fabiano Fidêncio --- .../local-build/kata-deploy-binaries.sh | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh index 19c85c1108..1a6b5dfd5e 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh @@ -8,6 +8,7 @@ set -o errexit set -o nounset set -o pipefail +set -o errtrace readonly project="kata-containers" @@ -196,6 +197,18 @@ handle_build() { tar tvf "${tarball_name}" } +silent_mode_error_trap() { + local stdout="$1" + local stderr="$2" + local t="$3" + local log_file="$4" + exec 1>&${stdout} + exec 2>&${stderr} + error "Failed to build: $t, logs:" + cat "${log_file}" + exit 1 +} + main() { local build_targets local silent @@ -248,11 +261,15 @@ main() { ( cd "${builddir}" if [ "${silent}" == true ]; then - if ! handle_build "${t}" &>"$log_file"; then - error "Failed to build: $t, logs:" - cat "${log_file}" - exit 1 - fi + local stdout + local stderr + # Save stdout and stderr, to be restored + # by silent_mode_error_trap() in case of + # build failure. + exec {stdout}>&1 + exec {stderr}>&2 + trap "silent_mode_error_trap $stdout $stderr $t \"$log_file\"" ERR + handle_build "${t}" &>"$log_file" else handle_build "${t}" fi From a37d4e538f15b8552f488673b2d5c83649e98e2d Mon Sep 17 00:00:00 2001 From: Feng Wang Date: Mon, 11 Apr 2022 18:44:23 -0700 Subject: [PATCH 10/29] agent: best-effort removing mount point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit During container exit, the agent tries to remove all the mount point directories, which can fail if it's a readonly filesytem (e.g. device mapper). This commit ignores the removal failure and logs a warning message. Fixes: #4043 Signed-off-by: Feng Wang (cherry picked from commit aabcebbf58ea2d4a3ec1e681e19b236fefa38088) Signed-off-by: Fabiano Fidêncio --- src/agent/src/sandbox.rs | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/agent/src/sandbox.rs b/src/agent/src/sandbox.rs index f1f60fbb91..78e2305f21 100644 --- a/src/agent/src/sandbox.rs +++ b/src/agent/src/sandbox.rs @@ -149,7 +149,12 @@ impl Sandbox { pub fn remove_sandbox_storage(&self, path: &str) -> Result<()> { let mounts = vec![path.to_string()]; remove_mounts(&mounts)?; - fs::remove_dir_all(path).context(format!("failed to remove dir {:?}", path))?; + // "remove_dir" will fail if the mount point is backed by a read-only filesystem. + // This is the case with the device mapper snapshotter, where we mount the block device directly + // at the underlying sandbox path which was provided from the base RO kataShared path from the host. + if let Err(err) = fs::remove_dir(path) { + warn!(self.logger, "failed to remove dir {}, {:?}", path, err); + } Ok(()) } @@ -562,19 +567,8 @@ mod tests { .remove_sandbox_storage(invalid_dir.to_str().unwrap()) .is_err()); - // Now, create a double mount as this guarantees the directory cannot - // be deleted after the first umount. - for _i in 0..2 { - assert!(bind_mount(srcdir_path, destdir_path, &logger).is_ok()); - } + assert!(bind_mount(srcdir_path, destdir_path, &logger).is_ok()); - assert!( - s.remove_sandbox_storage(destdir_path).is_err(), - "Expect fail as deletion cannot happen due to the second mount." - ); - - // This time it should work as the previous two calls have undone the double - // mount. assert!(s.remove_sandbox_storage(destdir_path).is_ok()); } From 8242cfd2be9c52ef6ccae8ce3fad4dcfac48db7a Mon Sep 17 00:00:00 2001 From: Francesco Giudici Date: Thu, 7 Apr 2022 11:18:46 +0200 Subject: [PATCH 11/29] kata-monitor: update the hrefs in the debug/pprof index page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kata-monitor allows to get data profiles from the kata shim instances running on the same node by acting as a proxy (e.g., http://$NODE_ADDRESS:8090/debug/pprof/?sandbox=$MYSANDBOXID). In order to proxy the requests and the responses to the right shim, kata-monitor requires to pass the sandbox id via a query string in the url. The profiling index page proxied by kata-monitor contains the link to all the data profiles available. All the links anyway do not contain the sandbox id included in the request: the links result then broken when accessed through kata-monitor. This happens because the profiling index page comes from the kata shim, which will not include the query string provided in the http request. Let's add on-the-fly the sandbox id in each href tag returned by the kata shim index page before providing the proxied page. Fixes: #4054 Signed-off-by: Francesco Giudici (cherry picked from commit 86977ff7809ce59c66fe7c7bf506a8bbf4b77a20) Signed-off-by: Fabiano Fidêncio --- src/runtime/pkg/kata-monitor/pprof.go | 54 ++++++++-- src/runtime/pkg/kata-monitor/pprof_test.go | 117 +++++++++++++++++++++ 2 files changed, 163 insertions(+), 8 deletions(-) create mode 100644 src/runtime/pkg/kata-monitor/pprof_test.go diff --git a/src/runtime/pkg/kata-monitor/pprof.go b/src/runtime/pkg/kata-monitor/pprof.go index 5dac2da03c..62ed70c2e7 100644 --- a/src/runtime/pkg/kata-monitor/pprof.go +++ b/src/runtime/pkg/kata-monitor/pprof.go @@ -10,6 +10,8 @@ import ( "io" "net" "net/http" + "regexp" + "strings" cdshim "github.com/containerd/containerd/runtime/v2/shim" @@ -33,7 +35,13 @@ func (km *KataMonitor) composeSocketAddress(r *http.Request) (string, error) { return shim.SocketAddress(sandbox), nil } -func (km *KataMonitor) proxyRequest(w http.ResponseWriter, r *http.Request) { +func (km *KataMonitor) proxyRequest(w http.ResponseWriter, r *http.Request, + proxyResponse func(req *http.Request, w io.Writer, r io.Reader) error) { + + if proxyResponse == nil { + proxyResponse = copyResponse + } + w.Header().Set("X-Content-Type-Options", "nosniff") socketAddress, err := km.composeSocketAddress(r) @@ -73,38 +81,68 @@ func (km *KataMonitor) proxyRequest(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Disposition", contentDisposition) } - io.Copy(w, output) + err = proxyResponse(r, w, output) + if err != nil { + monitorLog.WithError(err).Errorf("failed proxying %s from %s", uri, socketAddress) + serveError(w, http.StatusInternalServerError, "error retrieving resource") + } } // ExpvarHandler handles other `/debug/vars` requests func (km *KataMonitor) ExpvarHandler(w http.ResponseWriter, r *http.Request) { - km.proxyRequest(w, r) + km.proxyRequest(w, r, nil) } // PprofIndex handles other `/debug/pprof/` requests func (km *KataMonitor) PprofIndex(w http.ResponseWriter, r *http.Request) { - km.proxyRequest(w, r) + if len(strings.TrimPrefix(r.URL.Path, "/debug/pprof/")) == 0 { + km.proxyRequest(w, r, copyResponseAddingSandboxIdToHref) + } else { + km.proxyRequest(w, r, nil) + } } // PprofCmdline handles other `/debug/cmdline` requests func (km *KataMonitor) PprofCmdline(w http.ResponseWriter, r *http.Request) { - km.proxyRequest(w, r) + km.proxyRequest(w, r, nil) } // PprofProfile handles other `/debug/profile` requests func (km *KataMonitor) PprofProfile(w http.ResponseWriter, r *http.Request) { - km.proxyRequest(w, r) + km.proxyRequest(w, r, nil) } // PprofSymbol handles other `/debug/symbol` requests func (km *KataMonitor) PprofSymbol(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") - km.proxyRequest(w, r) + km.proxyRequest(w, r, nil) } // PprofTrace handles other `/debug/trace` requests func (km *KataMonitor) PprofTrace(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Disposition", `attachment; filename="trace"`) - km.proxyRequest(w, r) + km.proxyRequest(w, r, nil) +} + +func copyResponse(req *http.Request, w io.Writer, r io.Reader) error { + _, err := io.Copy(w, r) + return err +} + +func copyResponseAddingSandboxIdToHref(req *http.Request, w io.Writer, r io.Reader) error { + sb, err := getSandboxIDFromReq(req) + if err != nil { + monitorLog.WithError(err).Warning("missing sandbox query in pprof url") + return copyResponse(req, w, r) + } + buf, err := io.ReadAll(r) + if err != nil { + return err + } + + re := regexp.MustCompile(``) + outHtml := re.ReplaceAllString(string(buf), fmt.Sprintf("", sb)) + w.Write([]byte(outHtml)) + return nil } diff --git a/src/runtime/pkg/kata-monitor/pprof_test.go b/src/runtime/pkg/kata-monitor/pprof_test.go new file mode 100644 index 0000000000..e02dc00b12 --- /dev/null +++ b/src/runtime/pkg/kata-monitor/pprof_test.go @@ -0,0 +1,117 @@ +// Copyright (c) 2022 Red Hat Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package katamonitor + +import ( + "bytes" + "net/http" + "net/url" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCopyResponseAddingSandboxIdToHref(t *testing.T) { + assert := assert.New(t) + + htmlIn := strings.NewReader(` + + +/debug/pprof/ + + + +/debug/pprof/
+
+Types of profiles available: + + + + + + + + + + + +
CountProfile
27allocs
0block
0cmdline
39goroutine
27heap
0mutex
0profile
10threadcreate
0trace
+full goroutine stack dump +
+

+Profile Descriptions: +

    +
  • allocs:
    A sampling of all past memory allocations
    • +
  • block:
    Stack traces that led to blocking on synchronization primitives
    • +
  • cmdline:
    The command line invocation of the current program
    • +
  • goroutine:
    Stack traces of all current goroutines
    • +
  • heap:
    A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.
    • +
  • mutex:
    Stack traces of holders of contended mutexes
    • +
  • profile:
    CPU profile. You can specify the duration in the seconds GET parameter. After you get the profile file, use the go tool pprof command to investigate the profile.
    • +
  • threadcreate:
    Stack traces that led to the creation of new OS threads
    • +
  • trace:
    A trace of execution of the current program. You can specify the duration in the seconds GET parameter. After you get the trace file, use the go tool trace command to investigate the trace.
    • +
+

+ +`) + + htmlExpected := bytes.NewBufferString(` + + +/debug/pprof/ + + + +/debug/pprof/
+
+Types of profiles available: + + + + + + + + + + + +
CountProfile
27allocs
0block
0cmdline
39goroutine
27heap
0mutex
0profile
10threadcreate
0trace
+full goroutine stack dump +
+

+Profile Descriptions: +

    +
  • allocs:
    A sampling of all past memory allocations
    • +
  • block:
    Stack traces that led to blocking on synchronization primitives
    • +
  • cmdline:
    The command line invocation of the current program
    • +
  • goroutine:
    Stack traces of all current goroutines
    • +
  • heap:
    A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.
    • +
  • mutex:
    Stack traces of holders of contended mutexes
    • +
  • profile:
    CPU profile. You can specify the duration in the seconds GET parameter. After you get the profile file, use the go tool pprof command to investigate the profile.
    • +
  • threadcreate:
    Stack traces that led to the creation of new OS threads
    • +
  • trace:
    A trace of execution of the current program. You can specify the duration in the seconds GET parameter. After you get the trace file, use the go tool trace command to investigate the trace.
    • +
+

+ +`) + + req := &http.Request{URL: &url.URL{RawQuery: "sandbox=1234567890"}} + buf := bytes.NewBuffer(nil) + copyResponseAddingSandboxIdToHref(req, buf, htmlIn) + assert.Equal(htmlExpected, buf) +} From b62cced7f4099204fcdde8a89bd9d6e828fb405a Mon Sep 17 00:00:00 2001 From: Zhuoyu Tie Date: Fri, 8 Apr 2022 17:06:08 +0800 Subject: [PATCH 12/29] runtime: no need to write virtiofsd error to log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scanner reads nothing from viriofsd stderr pipe, because param '--syslog' rediercts stderr to syslog. So there is no need to write scanner.Text() to kata log Fixes: #4063 Signed-off-by: Zhuoyu Tie (cherry picked from commit 6e79042aa06197e186ad9ba65d3c23522dce4944) Signed-off-by: Fabiano Fidêncio --- src/runtime/virtcontainers/virtiofsd.go | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/runtime/virtcontainers/virtiofsd.go b/src/runtime/virtcontainers/virtiofsd.go index 6ad0d01cfc..b4c09cfd29 100644 --- a/src/runtime/virtcontainers/virtiofsd.go +++ b/src/runtime/virtcontainers/virtiofsd.go @@ -6,7 +6,6 @@ package virtcontainers import ( - "bufio" "context" "fmt" "net" @@ -136,24 +135,14 @@ func (v *virtiofsd) Start(ctx context.Context, onQuit onQuitFunc) (int, error) { v.Logger().WithField("path", v.path).Info() v.Logger().WithField("args", strings.Join(args, " ")).Info() - stderr, err := cmd.StderrPipe() - if err != nil { - return pid, err - } if err = utils.StartCmd(cmd); err != nil { return pid, err } - // Monitor virtiofsd's stderr and stop sandbox if virtiofsd quits go func() { - scanner := bufio.NewScanner(stderr) - for scanner.Scan() { - v.Logger().WithField("source", "virtiofsd").Info(scanner.Text()) - } - v.Logger().Info("virtiofsd quits") - // Wait to release resources of virtiofsd process cmd.Process.Wait() + v.Logger().Info("virtiofsd quits") if onQuit != nil { onQuit() } From 97ad1d55ffa4a934f95915776166471ae8d3cd61 Mon Sep 17 00:00:00 2001 From: Yibo Zhuang Date: Wed, 6 Apr 2022 19:31:49 -0700 Subject: [PATCH 13/29] runtime: fsGroup support for direct-assigned volume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fsGroup will be specified by the fsGroup key in the direct-assign mountinfo metadate field. This will be set when invoking the kata-runtime binary and providing the key, value pair in the metadata field. Similarly, the fsGroupChangePolicy will also be provided in the mountinfo metadate field. Adding an extra fields FsGroup and FSGroupChangePolicy in the Mount construct for container mount which will be populated when creating block devices by parsing out the mountInfo.json. And in handleDeviceBlockVolume of the kata-agent client, it checks if the mount FSGroup is not nil, which indicates that fsGroup change is required in the guest, and will provide the FSGroup field in the protobuf to pass the value to the agent. Fixes #4018 Signed-off-by: Yibo Zhuang (cherry picked from commit 532d53977ec276ecbb298a9755439bd3d95dbebf) Signed-off-by: Fabiano Fidêncio --- src/runtime/pkg/direct-volume/utils.go | 19 +++++++++++++++ src/runtime/pkg/direct-volume/utils_test.go | 7 +++++- src/runtime/virtcontainers/container.go | 20 ++++++++++++++++ src/runtime/virtcontainers/kata_agent.go | 16 +++++++++++++ src/runtime/virtcontainers/kata_agent_test.go | 23 +++++++++++++++++++ src/runtime/virtcontainers/mount.go | 10 ++++++++ 6 files changed, 94 insertions(+), 1 deletion(-) diff --git a/src/runtime/pkg/direct-volume/utils.go b/src/runtime/pkg/direct-volume/utils.go index 124e8b4ab5..8520ddd35e 100644 --- a/src/runtime/pkg/direct-volume/utils.go +++ b/src/runtime/pkg/direct-volume/utils.go @@ -17,6 +17,25 @@ import ( const ( mountInfoFileName = "mountInfo.json" + + FSGroupMetadataKey = "fsGroup" + FSGroupChangePolicyMetadataKey = "fsGroupChangePolicy" +) + +// FSGroupChangePolicy holds policies that will be used for applying fsGroup to a volume. +// This type and the allowed values are tracking the PodFSGroupChangePolicy defined in +// https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/api/core/v1/types.go +// It is up to the client using the direct-assigned volume feature (e.g. CSI drivers) to determine +// the optimal setting for this change policy (i.e. from Pod spec or assuming volume ownership +// based on the storage offering). +type FSGroupChangePolicy string + +const ( + // FSGroupChangeAlways indicates that volume's ownership should always be changed. + FSGroupChangeAlways FSGroupChangePolicy = "Always" + // FSGroupChangeOnRootMismatch indicates that volume's ownership will be changed + // only when ownership of root directory does not match with the desired group id. + FSGroupChangeOnRootMismatch FSGroupChangePolicy = "OnRootMismatch" ) var kataDirectVolumeRootPath = "/run/kata-containers/shared/direct-volumes" diff --git a/src/runtime/pkg/direct-volume/utils_test.go b/src/runtime/pkg/direct-volume/utils_test.go index 0fa8abb1c7..cf8884ff04 100644 --- a/src/runtime/pkg/direct-volume/utils_test.go +++ b/src/runtime/pkg/direct-volume/utils_test.go @@ -27,7 +27,11 @@ func TestAdd(t *testing.T) { VolumeType: "block", Device: "/dev/sda", FsType: "ext4", - Options: []string{"journal_dev", "noload"}, + Metadata: map[string]string{ + FSGroupMetadataKey: "3000", + FSGroupChangePolicyMetadataKey: string(FSGroupChangeOnRootMismatch), + }, + Options: []string{"journal_dev", "noload"}, } buf, err := json.Marshal(actual) assert.Nil(t, err) @@ -41,6 +45,7 @@ func TestAdd(t *testing.T) { assert.Equal(t, expected.Device, actual.Device) assert.Equal(t, expected.FsType, actual.FsType) assert.Equal(t, expected.Options, actual.Options) + assert.Equal(t, expected.Metadata, actual.Metadata) _, err = os.Stat(filepath.Join(kataDirectVolumeRootPath, b64.URLEncoding.EncodeToString([]byte(volumePath)))) assert.Nil(t, err) diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index d82124c1d6..4db6403ffa 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -639,6 +639,26 @@ func (c *Container) createBlockDevices(ctx context.Context) error { c.mounts[i].Type = mntInfo.FsType c.mounts[i].Options = mntInfo.Options c.mounts[i].ReadOnly = readonly + + for key, value := range mntInfo.Metadata { + switch key { + case volume.FSGroupMetadataKey: + gid, err := strconv.Atoi(value) + if err != nil { + c.Logger().WithError(err).Errorf("invalid group id value %s provided for key %s", value, volume.FSGroupMetadataKey) + continue + } + c.mounts[i].FSGroup = &gid + case volume.FSGroupChangePolicyMetadataKey: + if _, exists := mntInfo.Metadata[volume.FSGroupMetadataKey]; !exists { + c.Logger().Errorf("%s specified without provding the group id with key %s", volume.FSGroupChangePolicyMetadataKey, volume.FSGroupMetadataKey) + continue + } + c.mounts[i].FSGroupChangePolicy = volume.FSGroupChangePolicy(value) + default: + c.Logger().Warnf("Ignoring unsupported direct-assignd volume metadata key: %s, value: %s", key, value) + } + } } var stat unix.Stat_t diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 834a12b55e..04619efde6 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -19,6 +19,7 @@ import ( "time" "github.com/docker/go-units" + volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol" "github.com/kata-containers/kata-containers/src/runtime/pkg/uuid" @@ -167,6 +168,15 @@ func getPagesizeFromOpt(fsOpts []string) string { return "" } +func getFSGroupChangePolicy(policy volume.FSGroupChangePolicy) pbTypes.FSGroupChangePolicy { + switch policy { + case volume.FSGroupChangeOnRootMismatch: + return pbTypes.FSGroupChangePolicy_OnRootMismatch + default: + return pbTypes.FSGroupChangePolicy_Always + } +} + // Shared path handling: // 1. create three directories for each sandbox: // -. /run/kata-containers/shared/sandboxes/$sbx_id/mounts/, a directory to hold all host/guest shared mounts @@ -1468,6 +1478,12 @@ func (k *kataAgent) handleDeviceBlockVolume(c *Container, m Mount, device api.De if len(vol.Options) == 0 { vol.Options = m.Options } + if m.FSGroup != nil { + vol.FsGroup = &grpc.FSGroup{ + GroupId: uint32(*m.FSGroup), + GroupChangePolicy: getFSGroupChangePolicy(m.FSGroupChangePolicy), + } + } return vol, nil } diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index 1dbfe45698..3e3b5b6c65 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/assert" "code.cloudfoundry.org/bytefmt" + volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers" @@ -234,6 +235,7 @@ func TestHandleLocalStorage(t *testing.T) { } func TestHandleDeviceBlockVolume(t *testing.T) { + var gid = 2000 k := kataAgent{} // nolint: govet @@ -315,6 +317,27 @@ func TestHandleDeviceBlockVolume(t *testing.T) { Source: testSCSIAddr, }, }, + { + BlockDeviceDriver: config.VirtioBlock, + inputMount: Mount{ + FSGroup: &gid, + FSGroupChangePolicy: volume.FSGroupChangeOnRootMismatch, + }, + inputDev: &drivers.BlockDevice{ + BlockDrive: &config.BlockDrive{ + PCIPath: testPCIPath, + VirtPath: testVirtPath, + }, + }, + resultVol: &pb.Storage{ + Driver: kataBlkDevType, + Source: testPCIPath.String(), + FsGroup: &pb.FSGroup{ + GroupId: uint32(gid), + GroupChangePolicy: pbTypes.FSGroupChangePolicy_OnRootMismatch, + }, + }, + }, } for _, test := range tests { diff --git a/src/runtime/virtcontainers/mount.go b/src/runtime/virtcontainers/mount.go index 0e83bc6899..5e75826199 100644 --- a/src/runtime/virtcontainers/mount.go +++ b/src/runtime/virtcontainers/mount.go @@ -14,6 +14,7 @@ import ( "syscall" merr "github.com/hashicorp/go-multierror" + volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" "github.com/pkg/errors" @@ -325,6 +326,7 @@ func bindMountContainerRootfs(ctx context.Context, shareDir, cid, cRootFs string } // Mount describes a container mount. +// nolint: govet type Mount struct { // Source is the source of the mount. Source string @@ -352,6 +354,14 @@ type Mount struct { // ReadOnly specifies if the mount should be read only or not ReadOnly bool + + // FSGroup a group ID that the group ownership of the files for the mounted volume + // will need to be changed when set. + FSGroup *int + + // FSGroupChangePolicy specifies the policy that will be used when applying + // group id ownership change for a volume. + FSGroupChangePolicy volume.FSGroupChangePolicy } func isSymlink(path string) bool { From 01cd58094ef05b14d27a25c39b2cf723e2b4f0f6 Mon Sep 17 00:00:00 2001 From: Yibo Zhuang Date: Wed, 6 Apr 2022 19:31:13 -0700 Subject: [PATCH 14/29] proto: fsGroup support for direct-assigned volume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change adds two fields to the Storage pb FSGroup which is a group id that the runtime specifies to indicate to the agent to perform a chown of the mounted volume to the specified group id after mounting is complete in the guest. FSGroupChangePolicy which is a policy to indicate whether to always perform the group id ownership change or only if the root directory group id does not match with the desired group id. These two fields will allow CSI plugins to indicate to Kata that after the block device is mounted in the guest, group id ownership change should be performed on that volume. Fixes #4018 Signed-off-by: Yibo Zhuang (cherry picked from commit 6a47b82c812e1b63b1082f9976729093fe7d8c03) Signed-off-by: Fabiano Fidêncio --- src/libs/protocols/protos/agent.proto | 16 +- src/libs/protocols/protos/types.proto | 9 + .../pkg/agent/protocols/grpc/agent.pb.go | 688 ++++++++++++------ .../pkg/agent/protocols/types.pb.go | 96 ++- 4 files changed, 564 insertions(+), 245 deletions(-) diff --git a/src/libs/protocols/protos/agent.proto b/src/libs/protocols/protos/agent.proto index d0de9cf390..13a9094a8b 100644 --- a/src/libs/protocols/protos/agent.proto +++ b/src/libs/protocols/protos/agent.proto @@ -399,6 +399,17 @@ message SetGuestDateTimeRequest { int64 Usec = 2; } +// FSGroup consists of the group id and group ownership change policy +// that a volume should have its ownership changed to. +message FSGroup { + // GroupID is the ID that the group ownership of the + // files in the mounted volume will need to be changed to. + uint32 group_id = 2; + // GroupChangePolicy specifies the policy for applying group id + // ownership change on a mounted volume. + types.FSGroupChangePolicy group_change_policy = 3; +} + // Storage represents both the rootfs of the container, and any volume that // could have been defined through the Mount list of the OCI specification. message Storage { @@ -422,11 +433,14 @@ message Storage { // device, "9p" for shared filesystem, or "tmpfs" for shared /dev/shm. string fstype = 4; // Options describes the additional options that might be needed to - // mount properly the storage filesytem. + // mount properly the storage filesystem. repeated string options = 5; // MountPoint refers to the path where the storage should be mounted // inside the VM. string mount_point = 6; + // FSGroup consists of the group ID and group ownership change policy + // that the mounted volume must have its group ID changed to when specified. + FSGroup fs_group = 7; } // Device represents only the devices that could have been defined through the diff --git a/src/libs/protocols/protos/types.proto b/src/libs/protocols/protos/types.proto index f2586ef915..2ff6b7ab7e 100644 --- a/src/libs/protocols/protos/types.proto +++ b/src/libs/protocols/protos/types.proto @@ -16,6 +16,15 @@ enum IPFamily { v6 = 1; } +// FSGroupChangePolicy defines the policy for applying group id ownership change on a mounted volume. +enum FSGroupChangePolicy { + // Always indicates that the volume ownership will always be changed. + Always = 0; + // OnRootMismatch indicates that the volume ownership will be changed only + // when the ownership of the root directory does not match with the expected group id for the volume. + OnRootMismatch = 1; +} + message IPAddress { IPFamily family = 1; string address = 2; diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go index 437c9c8174..3a3e78504b 100644 --- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go +++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go @@ -1988,6 +1988,52 @@ func (m *SetGuestDateTimeRequest) XXX_DiscardUnknown() { var xxx_messageInfo_SetGuestDateTimeRequest proto.InternalMessageInfo +// FSGroup consists of the group id and group ownership change policy +// that a volume should have its ownership changed to. +type FSGroup struct { + // GroupID is the ID that the group ownership of the + // files in the mounted volume will need to be changed to. + GroupId uint32 `protobuf:"varint,2,opt,name=group_id,json=groupId,proto3" json:"group_id,omitempty"` + // GroupChangePolicy specifies the policy for applying group id + // ownership change on a mounted volume. + GroupChangePolicy protocols.FSGroupChangePolicy `protobuf:"varint,3,opt,name=group_change_policy,json=groupChangePolicy,proto3,enum=types.FSGroupChangePolicy" json:"group_change_policy,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *FSGroup) Reset() { *m = FSGroup{} } +func (*FSGroup) ProtoMessage() {} +func (*FSGroup) Descriptor() ([]byte, []int) { + return fileDescriptor_712ce9a559fda969, []int{47} +} +func (m *FSGroup) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *FSGroup) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_FSGroup.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *FSGroup) XXX_Merge(src proto.Message) { + xxx_messageInfo_FSGroup.Merge(m, src) +} +func (m *FSGroup) XXX_Size() int { + return m.Size() +} +func (m *FSGroup) XXX_DiscardUnknown() { + xxx_messageInfo_FSGroup.DiscardUnknown(m) +} + +var xxx_messageInfo_FSGroup proto.InternalMessageInfo + // Storage represents both the rootfs of the container, and any volume that // could have been defined through the Mount list of the OCI specification. type Storage struct { @@ -2011,11 +2057,14 @@ type Storage struct { // device, "9p" for shared filesystem, or "tmpfs" for shared /dev/shm. Fstype string `protobuf:"bytes,4,opt,name=fstype,proto3" json:"fstype,omitempty"` // Options describes the additional options that might be needed to - // mount properly the storage filesytem. + // mount properly the storage filesystem. Options []string `protobuf:"bytes,5,rep,name=options,proto3" json:"options,omitempty"` // MountPoint refers to the path where the storage should be mounted // inside the VM. - MountPoint string `protobuf:"bytes,6,opt,name=mount_point,json=mountPoint,proto3" json:"mount_point,omitempty"` + MountPoint string `protobuf:"bytes,6,opt,name=mount_point,json=mountPoint,proto3" json:"mount_point,omitempty"` + // FSGroup consists of the group ID and group ownership change policy + // that the mounted volume must have its group ID changed to when specified. + FsGroup *FSGroup `protobuf:"bytes,7,opt,name=fs_group,json=fsGroup,proto3" json:"fs_group,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2024,7 +2073,7 @@ type Storage struct { func (m *Storage) Reset() { *m = Storage{} } func (*Storage) ProtoMessage() {} func (*Storage) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{47} + return fileDescriptor_712ce9a559fda969, []int{48} } func (m *Storage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2095,7 +2144,7 @@ type Device struct { func (m *Device) Reset() { *m = Device{} } func (*Device) ProtoMessage() {} func (*Device) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{48} + return fileDescriptor_712ce9a559fda969, []int{49} } func (m *Device) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2136,7 +2185,7 @@ type StringUser struct { func (m *StringUser) Reset() { *m = StringUser{} } func (*StringUser) ProtoMessage() {} func (*StringUser) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{49} + return fileDescriptor_712ce9a559fda969, []int{50} } func (m *StringUser) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2193,7 +2242,7 @@ type CopyFileRequest struct { func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } func (*CopyFileRequest) ProtoMessage() {} func (*CopyFileRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{50} + return fileDescriptor_712ce9a559fda969, []int{51} } func (m *CopyFileRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2231,7 +2280,7 @@ type GetOOMEventRequest struct { func (m *GetOOMEventRequest) Reset() { *m = GetOOMEventRequest{} } func (*GetOOMEventRequest) ProtoMessage() {} func (*GetOOMEventRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{51} + return fileDescriptor_712ce9a559fda969, []int{52} } func (m *GetOOMEventRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2270,7 +2319,7 @@ type OOMEvent struct { func (m *OOMEvent) Reset() { *m = OOMEvent{} } func (*OOMEvent) ProtoMessage() {} func (*OOMEvent) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{52} + return fileDescriptor_712ce9a559fda969, []int{53} } func (m *OOMEvent) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2309,7 +2358,7 @@ type AddSwapRequest struct { func (m *AddSwapRequest) Reset() { *m = AddSwapRequest{} } func (*AddSwapRequest) ProtoMessage() {} func (*AddSwapRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{53} + return fileDescriptor_712ce9a559fda969, []int{54} } func (m *AddSwapRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2347,7 +2396,7 @@ type GetMetricsRequest struct { func (m *GetMetricsRequest) Reset() { *m = GetMetricsRequest{} } func (*GetMetricsRequest) ProtoMessage() {} func (*GetMetricsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{54} + return fileDescriptor_712ce9a559fda969, []int{55} } func (m *GetMetricsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2386,7 +2435,7 @@ type Metrics struct { func (m *Metrics) Reset() { *m = Metrics{} } func (*Metrics) ProtoMessage() {} func (*Metrics) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{55} + return fileDescriptor_712ce9a559fda969, []int{56} } func (m *Metrics) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2426,7 +2475,7 @@ type VolumeStatsRequest struct { func (m *VolumeStatsRequest) Reset() { *m = VolumeStatsRequest{} } func (*VolumeStatsRequest) ProtoMessage() {} func (*VolumeStatsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{56} + return fileDescriptor_712ce9a559fda969, []int{57} } func (m *VolumeStatsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2467,7 +2516,7 @@ type ResizeVolumeRequest struct { func (m *ResizeVolumeRequest) Reset() { *m = ResizeVolumeRequest{} } func (*ResizeVolumeRequest) ProtoMessage() {} func (*ResizeVolumeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_712ce9a559fda969, []int{57} + return fileDescriptor_712ce9a559fda969, []int{58} } func (m *ResizeVolumeRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2546,6 +2595,7 @@ func init() { proto.RegisterType((*GuestDetailsResponse)(nil), "grpc.GuestDetailsResponse") proto.RegisterType((*MemHotplugByProbeRequest)(nil), "grpc.MemHotplugByProbeRequest") proto.RegisterType((*SetGuestDateTimeRequest)(nil), "grpc.SetGuestDateTimeRequest") + proto.RegisterType((*FSGroup)(nil), "grpc.FSGroup") proto.RegisterType((*Storage)(nil), "grpc.Storage") proto.RegisterType((*Device)(nil), "grpc.Device") proto.RegisterType((*StringUser)(nil), "grpc.StringUser") @@ -2564,198 +2614,203 @@ func init() { } var fileDescriptor_712ce9a559fda969 = []byte{ - // 3055 bytes of a gzipped FileDescriptorProto + // 3127 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x1a, 0xcb, 0x72, 0x24, 0x47, 0xd1, 0xf3, 0x90, 0x66, 0x26, 0xe7, 0xa5, 0x69, 0x69, 0xb5, 0xb3, 0x63, 0x5b, 0xac, 0x7b, 0xed, - 0xf5, 0xda, 0xc6, 0x92, 0xbd, 0x76, 0xb0, 0x7e, 0x84, 0x59, 0x24, 0xad, 0x2c, 0xc9, 0xb6, 0xbc, - 0x43, 0xcb, 0xc2, 0x04, 0x04, 0x74, 0xb4, 0xba, 0x6b, 0x47, 0x65, 0x4d, 0x77, 0xb5, 0xab, 0xab, - 0xb5, 0x92, 0x89, 0x20, 0x38, 0xc1, 0x8d, 0x23, 0x37, 0x7e, 0x80, 0xe0, 0xc6, 0x91, 0x0b, 0x07, - 0x0e, 0x0e, 0x4e, 0x1c, 0x39, 0x11, 0x78, 0x3f, 0x81, 0x2f, 0x20, 0xea, 0xd5, 0x5d, 0x3d, 0x0f, - 0x19, 0x14, 0x1b, 0xc1, 0x65, 0xa2, 0x33, 0x2b, 0x2b, 0x5f, 0x55, 0x99, 0x95, 0x59, 0x35, 0x30, - 0x1c, 0x61, 0x76, 0x92, 0x1e, 0xaf, 0xfb, 0x24, 0xdc, 0x38, 0xf5, 0x98, 0xf7, 0xba, 0x4f, 0x22, - 0xe6, 0xe1, 0x08, 0xd1, 0x64, 0x0a, 0x4e, 0xa8, 0xbf, 0x31, 0xc6, 0xc7, 0xc9, 0x46, 0x4c, 0x09, - 0x23, 0x3e, 0x19, 0xab, 0xaf, 0x64, 0xc3, 0x1b, 0xa1, 0x88, 0xad, 0x0b, 0xc0, 0xaa, 0x8e, 0x68, - 0xec, 0x0f, 0x1a, 0xc4, 0xc7, 0x12, 0x31, 0x68, 0xf8, 0x89, 0xfe, 0x6c, 0xb2, 0x8b, 0x18, 0x25, - 0x0a, 0x78, 0x76, 0x44, 0xc8, 0x68, 0x8c, 0x24, 0x8f, 0xe3, 0xf4, 0xd1, 0x06, 0x0a, 0x63, 0x76, - 0x21, 0x07, 0xed, 0xdf, 0x97, 0x61, 0x75, 0x9b, 0x22, 0x8f, 0xa1, 0x6d, 0xad, 0x80, 0x83, 0xbe, - 0x4c, 0x51, 0xc2, 0xac, 0x17, 0xa0, 0x95, 0x29, 0xe5, 0xe2, 0xa0, 0x5f, 0xba, 0x59, 0xba, 0xd3, - 0x70, 0x9a, 0x19, 0x6e, 0x3f, 0xb0, 0xae, 0x43, 0x0d, 0x9d, 0x23, 0x9f, 0x8f, 0x96, 0xc5, 0xe8, - 0x22, 0x07, 0xf7, 0x03, 0xeb, 0x4d, 0x68, 0x26, 0x8c, 0xe2, 0x68, 0xe4, 0xa6, 0x09, 0xa2, 0xfd, - 0xca, 0xcd, 0xd2, 0x9d, 0xe6, 0xdd, 0xa5, 0x75, 0xae, 0xf2, 0xfa, 0xa1, 0x18, 0x38, 0x4a, 0x10, - 0x75, 0x20, 0xc9, 0xbe, 0xad, 0xdb, 0x50, 0x0b, 0xd0, 0x19, 0xf6, 0x51, 0xd2, 0xaf, 0xde, 0xac, - 0xdc, 0x69, 0xde, 0x6d, 0x49, 0xf2, 0x07, 0x02, 0xe9, 0xe8, 0x41, 0xeb, 0x15, 0xa8, 0x27, 0x8c, - 0x50, 0x6f, 0x84, 0x92, 0xfe, 0x82, 0x20, 0x6c, 0x6b, 0xbe, 0x02, 0xeb, 0x64, 0xc3, 0xd6, 0x73, - 0x50, 0x79, 0xb8, 0xbd, 0xdf, 0x5f, 0x14, 0xd2, 0x41, 0x51, 0xc5, 0xc8, 0x77, 0x38, 0xda, 0xba, - 0x05, 0xed, 0xc4, 0x8b, 0x82, 0x63, 0x72, 0xee, 0xc6, 0x38, 0x88, 0x92, 0x7e, 0xed, 0x66, 0xe9, - 0x4e, 0xdd, 0x69, 0x29, 0xe4, 0x90, 0xe3, 0xec, 0xf7, 0xe0, 0xda, 0x21, 0xf3, 0x28, 0xbb, 0x82, - 0x77, 0xec, 0x23, 0x58, 0x75, 0x50, 0x48, 0xce, 0xae, 0xe4, 0xda, 0x3e, 0xd4, 0x18, 0x0e, 0x11, - 0x49, 0x99, 0x70, 0x6d, 0xdb, 0xd1, 0xa0, 0xfd, 0xc7, 0x12, 0x58, 0x3b, 0xe7, 0xc8, 0x1f, 0x52, - 0xe2, 0xa3, 0x24, 0xf9, 0x3f, 0x2d, 0xd7, 0xcb, 0x50, 0x8b, 0xa5, 0x02, 0xfd, 0xaa, 0x20, 0x57, - 0xab, 0xa0, 0xb5, 0xd2, 0xa3, 0xf6, 0x17, 0xb0, 0x72, 0x88, 0x47, 0x91, 0x37, 0x7e, 0x8a, 0xfa, - 0xae, 0xc2, 0x62, 0x22, 0x78, 0x0a, 0x55, 0xdb, 0x8e, 0x82, 0xec, 0x21, 0x58, 0x9f, 0x7b, 0x98, - 0x3d, 0x3d, 0x49, 0xf6, 0xeb, 0xb0, 0x5c, 0xe0, 0x98, 0xc4, 0x24, 0x4a, 0x90, 0x50, 0x80, 0x79, - 0x2c, 0x4d, 0x04, 0xb3, 0x05, 0x47, 0x41, 0x36, 0x81, 0xd5, 0xa3, 0x38, 0xb8, 0x62, 0x34, 0xdd, - 0x85, 0x06, 0x45, 0x09, 0x49, 0x29, 0x8f, 0x81, 0xb2, 0x70, 0xea, 0x8a, 0x74, 0xea, 0x27, 0x38, - 0x4a, 0xcf, 0x1d, 0x3d, 0xe6, 0xe4, 0x64, 0x6a, 0x7f, 0xb2, 0xe4, 0x2a, 0xfb, 0xf3, 0x3d, 0xb8, - 0x36, 0xf4, 0xd2, 0xe4, 0x2a, 0xba, 0xda, 0xef, 0xf3, 0xbd, 0x9d, 0xa4, 0xe1, 0x95, 0x26, 0xff, - 0xa1, 0x04, 0xf5, 0xed, 0x38, 0x3d, 0x4a, 0xbc, 0x11, 0xb2, 0xbe, 0x03, 0x4d, 0x46, 0x98, 0x37, - 0x76, 0x53, 0x0e, 0x0a, 0xf2, 0xaa, 0x03, 0x02, 0x25, 0x09, 0x5e, 0x80, 0x56, 0x8c, 0xa8, 0x1f, - 0xa7, 0x8a, 0xa2, 0x7c, 0xb3, 0x72, 0xa7, 0xea, 0x34, 0x25, 0x4e, 0x92, 0xac, 0xc3, 0xb2, 0x18, - 0x73, 0x71, 0xe4, 0x9e, 0x22, 0x1a, 0xa1, 0x71, 0x48, 0x02, 0x24, 0x36, 0x47, 0xd5, 0xe9, 0x89, - 0xa1, 0xfd, 0xe8, 0xe3, 0x6c, 0xc0, 0x7a, 0x15, 0x7a, 0x19, 0x3d, 0xdf, 0xf1, 0x82, 0xba, 0x2a, - 0xa8, 0xbb, 0x8a, 0xfa, 0x48, 0xa1, 0xed, 0x5f, 0x42, 0xe7, 0xb3, 0x13, 0x4a, 0x18, 0x1b, 0xe3, - 0x68, 0xf4, 0xc0, 0x63, 0x1e, 0x0f, 0xcd, 0x18, 0x51, 0x4c, 0x82, 0x44, 0x69, 0xab, 0x41, 0xeb, - 0x35, 0xe8, 0x31, 0x49, 0x8b, 0x02, 0x57, 0xd3, 0x94, 0x05, 0xcd, 0x52, 0x36, 0x30, 0x54, 0xc4, - 0x2f, 0x41, 0x27, 0x27, 0xe6, 0xc1, 0xad, 0xf4, 0x6d, 0x67, 0xd8, 0xcf, 0x70, 0x88, 0xec, 0x33, - 0xe1, 0x2b, 0xb1, 0xc8, 0xd6, 0x6b, 0xd0, 0xc8, 0xfd, 0x50, 0x12, 0x3b, 0xa4, 0x23, 0x77, 0x88, - 0x76, 0xa7, 0x53, 0xcf, 0x9c, 0xf2, 0x01, 0x74, 0x59, 0xa6, 0xb8, 0x1b, 0x78, 0xcc, 0x2b, 0x6e, - 0xaa, 0xa2, 0x55, 0x4e, 0x87, 0x15, 0x60, 0xfb, 0x7d, 0x68, 0x0c, 0x71, 0x90, 0x48, 0xc1, 0x7d, - 0xa8, 0xf9, 0x29, 0xa5, 0x28, 0x62, 0xda, 0x64, 0x05, 0x5a, 0x2b, 0xb0, 0x30, 0xc6, 0x21, 0x66, - 0xca, 0x4c, 0x09, 0xd8, 0x04, 0xe0, 0x00, 0x85, 0x84, 0x5e, 0x08, 0x87, 0xad, 0xc0, 0x82, 0xb9, - 0xb8, 0x12, 0xb0, 0x9e, 0x85, 0x46, 0xe8, 0x9d, 0x67, 0x8b, 0xca, 0x47, 0xea, 0xa1, 0x77, 0x2e, - 0x95, 0xef, 0x43, 0xed, 0x91, 0x87, 0xc7, 0x7e, 0xc4, 0x94, 0x57, 0x34, 0x98, 0x0b, 0xac, 0x9a, - 0x02, 0xff, 0x5a, 0x86, 0xa6, 0x94, 0x28, 0x15, 0x5e, 0x81, 0x05, 0xdf, 0xf3, 0x4f, 0x32, 0x91, - 0x02, 0xb0, 0x6e, 0x6b, 0x45, 0xca, 0x66, 0x86, 0xcb, 0x35, 0xd5, 0xaa, 0x6d, 0x00, 0x24, 0x8f, - 0xbd, 0x58, 0xe9, 0x56, 0x99, 0x43, 0xdc, 0xe0, 0x34, 0x52, 0xdd, 0xb7, 0xa0, 0x25, 0xf7, 0x9d, - 0x9a, 0x52, 0x9d, 0x33, 0xa5, 0x29, 0xa9, 0xe4, 0xa4, 0x5b, 0xd0, 0x4e, 0x13, 0xe4, 0x9e, 0x60, - 0x44, 0x3d, 0xea, 0x9f, 0x5c, 0xf4, 0x17, 0xe4, 0x01, 0x94, 0x26, 0x68, 0x4f, 0xe3, 0xac, 0xbb, - 0xb0, 0xc0, 0x73, 0x4b, 0xd2, 0x5f, 0x14, 0x67, 0xdd, 0x73, 0x26, 0x4b, 0x61, 0xea, 0xba, 0xf8, - 0xdd, 0x89, 0x18, 0xbd, 0x70, 0x24, 0xe9, 0xe0, 0x1d, 0x80, 0x1c, 0x69, 0x2d, 0x41, 0xe5, 0x14, - 0x5d, 0xa8, 0x38, 0xe4, 0x9f, 0xdc, 0x39, 0x67, 0xde, 0x38, 0xd5, 0x5e, 0x97, 0xc0, 0x7b, 0xe5, - 0x77, 0x4a, 0xb6, 0x0f, 0xdd, 0xad, 0xf1, 0x29, 0x26, 0xc6, 0xf4, 0x15, 0x58, 0x08, 0xbd, 0x2f, - 0x08, 0xd5, 0x9e, 0x14, 0x80, 0xc0, 0xe2, 0x88, 0x50, 0xcd, 0x42, 0x00, 0x56, 0x07, 0xca, 0x24, - 0x16, 0xfe, 0x6a, 0x38, 0x65, 0x12, 0xe7, 0x82, 0xaa, 0x86, 0x20, 0xfb, 0x9f, 0x55, 0x80, 0x5c, - 0x8a, 0xe5, 0xc0, 0x00, 0x13, 0x37, 0x41, 0x94, 0x9f, 0xef, 0xee, 0xf1, 0x05, 0x43, 0x89, 0x4b, - 0x91, 0x9f, 0xd2, 0x04, 0x9f, 0xf1, 0xf5, 0xe3, 0x66, 0x5f, 0x93, 0x66, 0x4f, 0xe8, 0xe6, 0x5c, - 0xc7, 0xe4, 0x50, 0xce, 0xdb, 0xe2, 0xd3, 0x1c, 0x3d, 0xcb, 0xda, 0x87, 0x6b, 0x39, 0xcf, 0xc0, - 0x60, 0x57, 0xbe, 0x8c, 0xdd, 0x72, 0xc6, 0x2e, 0xc8, 0x59, 0xed, 0xc0, 0x32, 0x26, 0xee, 0x97, - 0x29, 0x4a, 0x0b, 0x8c, 0x2a, 0x97, 0x31, 0xea, 0x61, 0xf2, 0x43, 0x31, 0x21, 0x67, 0x33, 0x84, - 0x1b, 0x86, 0x95, 0x3c, 0xdc, 0x0d, 0x66, 0xd5, 0xcb, 0x98, 0xad, 0x66, 0x5a, 0xf1, 0x7c, 0x90, - 0x73, 0xfc, 0x08, 0x56, 0x31, 0x71, 0x1f, 0x7b, 0x98, 0x4d, 0xb2, 0x5b, 0xf8, 0x16, 0x23, 0xf9, - 0x89, 0x56, 0xe4, 0x25, 0x8d, 0x0c, 0x11, 0x1d, 0x15, 0x8c, 0x5c, 0xfc, 0x16, 0x23, 0x0f, 0xc4, - 0x84, 0x9c, 0xcd, 0x26, 0xf4, 0x30, 0x99, 0xd4, 0xa6, 0x76, 0x19, 0x93, 0x2e, 0x26, 0x45, 0x4d, - 0xb6, 0xa0, 0x97, 0x20, 0x9f, 0x11, 0x6a, 0x6e, 0x82, 0xfa, 0x65, 0x2c, 0x96, 0x14, 0x7d, 0xc6, - 0xc3, 0xfe, 0x29, 0xb4, 0xf6, 0xd2, 0x11, 0x62, 0xe3, 0xe3, 0x2c, 0x19, 0x3c, 0xb5, 0xfc, 0x63, - 0xff, 0xbb, 0x0c, 0xcd, 0xed, 0x11, 0x25, 0x69, 0x5c, 0xc8, 0xc9, 0x32, 0x48, 0x27, 0x73, 0xb2, - 0x20, 0x11, 0x39, 0x59, 0x12, 0xbf, 0x0d, 0xad, 0x50, 0x84, 0xae, 0xa2, 0x97, 0x79, 0xa8, 0x37, - 0x15, 0xd4, 0x4e, 0x33, 0x34, 0x92, 0xd9, 0x3a, 0x40, 0x8c, 0x83, 0x44, 0xcd, 0x91, 0xe9, 0xa8, - 0xab, 0xca, 0x2d, 0x9d, 0xa2, 0x9d, 0x46, 0x9c, 0x65, 0xeb, 0x37, 0xa1, 0x79, 0xcc, 0x9d, 0xa4, - 0x26, 0x14, 0x92, 0x51, 0xee, 0x3d, 0x07, 0x8e, 0xf3, 0x20, 0xdc, 0x83, 0xf6, 0x89, 0x74, 0x99, - 0x9a, 0x24, 0xf7, 0xd0, 0x2d, 0x65, 0x49, 0x6e, 0xef, 0xba, 0xe9, 0x59, 0xb9, 0x00, 0xad, 0x13, - 0x03, 0x35, 0x38, 0x84, 0xde, 0x14, 0xc9, 0x8c, 0x1c, 0x74, 0xc7, 0xcc, 0x41, 0xcd, 0xbb, 0x96, - 0x14, 0x64, 0xce, 0x34, 0xf3, 0xd2, 0x6f, 0xcb, 0xd0, 0xfa, 0x14, 0xb1, 0xc7, 0x84, 0x9e, 0x4a, - 0x7d, 0x2d, 0xa8, 0x46, 0x5e, 0x88, 0x14, 0x47, 0xf1, 0x6d, 0xdd, 0x80, 0x3a, 0x3d, 0x97, 0x09, - 0x44, 0xad, 0x67, 0x8d, 0x9e, 0x8b, 0xc4, 0x60, 0x3d, 0x0f, 0x40, 0xcf, 0xdd, 0xd8, 0xf3, 0x4f, - 0x91, 0xf2, 0x60, 0xd5, 0x69, 0xd0, 0xf3, 0xa1, 0x44, 0xf0, 0xad, 0x40, 0xcf, 0x5d, 0x44, 0x29, - 0xa1, 0x89, 0xca, 0x55, 0x75, 0x7a, 0xbe, 0x23, 0x60, 0x35, 0x37, 0xa0, 0x24, 0x8e, 0x51, 0x20, - 0x72, 0xb4, 0x98, 0xfb, 0x40, 0x22, 0xb8, 0x54, 0xa6, 0xa5, 0x2e, 0x4a, 0xa9, 0x2c, 0x97, 0xca, - 0x72, 0xa9, 0x35, 0x39, 0x93, 0x99, 0x52, 0x59, 0x26, 0xb5, 0x2e, 0xa5, 0x32, 0x43, 0x2a, 0xcb, - 0xa5, 0x36, 0xf4, 0x5c, 0x25, 0xd5, 0xfe, 0x4d, 0x09, 0x56, 0x27, 0x0b, 0x3f, 0x55, 0x9b, 0xbe, - 0x0d, 0x2d, 0x5f, 0xac, 0x57, 0x61, 0x4f, 0xf6, 0xa6, 0x56, 0xd2, 0x69, 0xfa, 0xc6, 0x36, 0xbe, - 0x07, 0xed, 0x48, 0x3a, 0x38, 0xdb, 0x9a, 0x95, 0x7c, 0x5d, 0x4c, 0xdf, 0x3b, 0xad, 0xc8, 0x80, - 0xec, 0x00, 0xac, 0xcf, 0x29, 0x66, 0xe8, 0x90, 0x51, 0xe4, 0x85, 0x4f, 0xa3, 0xba, 0xb7, 0xa0, - 0x2a, 0xaa, 0x15, 0xbe, 0x4c, 0x2d, 0x47, 0x7c, 0xdb, 0x2f, 0xc3, 0x72, 0x41, 0x8a, 0xb2, 0x75, - 0x09, 0x2a, 0x63, 0x14, 0x09, 0xee, 0x6d, 0x87, 0x7f, 0xda, 0x1e, 0xf4, 0x1c, 0xe4, 0x05, 0x4f, - 0x4f, 0x1b, 0x25, 0xa2, 0x92, 0x8b, 0xb8, 0x03, 0x96, 0x29, 0x42, 0xa9, 0xa2, 0xb5, 0x2e, 0x19, - 0x5a, 0x3f, 0x84, 0xde, 0xf6, 0x98, 0x24, 0xe8, 0x90, 0x05, 0x38, 0x7a, 0x1a, 0xed, 0xc8, 0x2f, - 0x60, 0xf9, 0x33, 0x76, 0xf1, 0x39, 0x67, 0x96, 0xe0, 0xaf, 0xd0, 0x53, 0xb2, 0x8f, 0x92, 0xc7, - 0xda, 0x3e, 0x4a, 0x1e, 0xf3, 0xe6, 0xc6, 0x27, 0xe3, 0x34, 0x8c, 0x44, 0x28, 0xb4, 0x1d, 0x05, - 0xd9, 0x5b, 0xd0, 0x92, 0x35, 0xf4, 0x01, 0x09, 0xd2, 0x31, 0x9a, 0x19, 0x83, 0x6b, 0x00, 0xb1, - 0x47, 0xbd, 0x10, 0x31, 0x44, 0xe5, 0x1e, 0x6a, 0x38, 0x06, 0xc6, 0xfe, 0x5d, 0x19, 0x56, 0xe4, - 0x7d, 0xc3, 0xa1, 0x6c, 0xb3, 0xb5, 0x09, 0x03, 0xa8, 0x9f, 0x90, 0x84, 0x19, 0x0c, 0x33, 0x98, - 0xab, 0xc8, 0xfb, 0x73, 0xc9, 0x8d, 0x7f, 0x16, 0x2e, 0x01, 0x2a, 0x97, 0x5f, 0x02, 0x4c, 0xb5, - 0xf9, 0xd5, 0xe9, 0x36, 0x9f, 0x47, 0x9b, 0x26, 0xc2, 0x32, 0xc6, 0x1b, 0x4e, 0x43, 0x61, 0xf6, - 0x03, 0xeb, 0x36, 0x74, 0x47, 0x5c, 0x4b, 0xf7, 0x84, 0x90, 0x53, 0x37, 0xf6, 0xd8, 0x89, 0x08, - 0xf5, 0x86, 0xd3, 0x16, 0xe8, 0x3d, 0x42, 0x4e, 0x87, 0x1e, 0x3b, 0xb1, 0xde, 0x85, 0x8e, 0x2a, - 0x03, 0x43, 0xe1, 0xa2, 0x44, 0x1d, 0x7e, 0x2a, 0x8a, 0x4c, 0xef, 0x39, 0xed, 0x53, 0x03, 0x4a, - 0xec, 0xeb, 0x70, 0xed, 0x01, 0x4a, 0x18, 0x25, 0x17, 0x45, 0xc7, 0xd8, 0xdf, 0x07, 0xd8, 0x8f, - 0x18, 0xa2, 0x8f, 0x3c, 0x1f, 0x25, 0xd6, 0x1b, 0x26, 0xa4, 0x8a, 0xa3, 0xa5, 0x75, 0x79, 0xdd, - 0x93, 0x0d, 0x38, 0x06, 0x8d, 0xbd, 0x0e, 0x8b, 0x0e, 0x49, 0x79, 0x3a, 0x7a, 0x51, 0x7f, 0xa9, - 0x79, 0x2d, 0x35, 0x4f, 0x20, 0x1d, 0x35, 0x66, 0xef, 0xe9, 0x16, 0x36, 0x67, 0xa7, 0x96, 0x68, - 0x1d, 0x1a, 0x58, 0xe3, 0x54, 0x56, 0x99, 0x16, 0x9d, 0x93, 0xd8, 0xef, 0xc3, 0xb2, 0xe4, 0x24, - 0x39, 0x6b, 0x36, 0x2f, 0xc2, 0x22, 0xd5, 0x6a, 0x94, 0xf2, 0x7b, 0x1e, 0x45, 0xa4, 0xc6, 0xb8, - 0x3f, 0x3e, 0xc1, 0x09, 0xcb, 0x0d, 0xd1, 0xfe, 0x58, 0x86, 0x1e, 0x1f, 0x28, 0xf0, 0xb4, 0x3f, - 0x84, 0xd6, 0xa6, 0x33, 0xfc, 0x14, 0xe1, 0xd1, 0xc9, 0x31, 0xcf, 0x9e, 0xdf, 0x2b, 0xc2, 0xca, - 0x60, 0x4b, 0x69, 0x6b, 0x0c, 0x39, 0x05, 0x3a, 0xfb, 0x23, 0x58, 0xdd, 0x0c, 0x02, 0x13, 0xa5, - 0xb5, 0x7e, 0x03, 0x1a, 0x91, 0xc1, 0xce, 0x38, 0xb3, 0x0a, 0xd4, 0x39, 0x91, 0xfd, 0x33, 0x58, - 0x7e, 0x18, 0x8d, 0x71, 0x84, 0xb6, 0x87, 0x47, 0x07, 0x28, 0xcb, 0x45, 0x16, 0x54, 0x79, 0xcd, - 0x26, 0x78, 0xd4, 0x1d, 0xf1, 0xcd, 0x83, 0x33, 0x3a, 0x76, 0xfd, 0x38, 0x4d, 0xd4, 0x65, 0xcf, - 0x62, 0x74, 0xbc, 0x1d, 0xa7, 0x09, 0x3f, 0x5c, 0x78, 0x71, 0x41, 0xa2, 0xf1, 0x85, 0x88, 0xd0, - 0xba, 0x53, 0xf3, 0xe3, 0xf4, 0x61, 0x34, 0xbe, 0xb0, 0xbf, 0x2b, 0x3a, 0x70, 0x84, 0x02, 0xc7, - 0x8b, 0x02, 0x12, 0x3e, 0x40, 0x67, 0x86, 0x84, 0xac, 0xdb, 0xd3, 0x99, 0xe8, 0xeb, 0x12, 0xb4, - 0x36, 0x47, 0x28, 0x62, 0x0f, 0x10, 0xf3, 0xf0, 0x58, 0x74, 0x74, 0x67, 0x88, 0x26, 0x98, 0x44, - 0x2a, 0xdc, 0x34, 0xc8, 0x1b, 0x72, 0x1c, 0x61, 0xe6, 0x06, 0x1e, 0x0a, 0x49, 0x24, 0xb8, 0xd4, - 0x1d, 0xe0, 0xa8, 0x07, 0x02, 0x63, 0xbd, 0x0c, 0x5d, 0x79, 0x19, 0xe7, 0x9e, 0x78, 0x51, 0x30, - 0xe6, 0x81, 0x5e, 0x11, 0xa1, 0xd9, 0x91, 0xe8, 0x3d, 0x85, 0xb5, 0x5e, 0x81, 0x25, 0x15, 0x86, - 0x39, 0x65, 0x55, 0x50, 0x76, 0x15, 0xbe, 0x40, 0x9a, 0xc6, 0x31, 0xa1, 0x2c, 0x71, 0x13, 0xe4, - 0xfb, 0x24, 0x8c, 0x55, 0x3b, 0xd4, 0xd5, 0xf8, 0x43, 0x89, 0xb6, 0x47, 0xb0, 0xbc, 0xcb, 0xed, - 0x54, 0x96, 0xe4, 0xdb, 0xaa, 0x13, 0xa2, 0xd0, 0x3d, 0x1e, 0x13, 0xff, 0xd4, 0xe5, 0xc9, 0x51, - 0x79, 0x98, 0x17, 0x5c, 0x5b, 0x1c, 0x79, 0x88, 0xbf, 0x12, 0x9d, 0x3f, 0xa7, 0x3a, 0x21, 0x2c, - 0x1e, 0xa7, 0x23, 0x37, 0xa6, 0xe4, 0x18, 0x29, 0x13, 0xbb, 0x21, 0x0a, 0xf7, 0x24, 0x7e, 0xc8, - 0xd1, 0xf6, 0x9f, 0x4b, 0xb0, 0x52, 0x94, 0xa4, 0x52, 0xfd, 0x06, 0xac, 0x14, 0x45, 0xa9, 0xe3, - 0x5f, 0x96, 0x97, 0x3d, 0x53, 0xa0, 0x2c, 0x04, 0xee, 0x41, 0x5b, 0x5c, 0xdd, 0xba, 0x81, 0xe4, - 0x54, 0x2c, 0x7a, 0xcc, 0x75, 0x71, 0x5a, 0x9e, 0xb9, 0x4a, 0xef, 0xc2, 0x0d, 0x65, 0xbe, 0x3b, - 0xad, 0xb6, 0xdc, 0x10, 0xab, 0x8a, 0xe0, 0x60, 0x42, 0xfb, 0x4f, 0xa0, 0x9f, 0xa3, 0xb6, 0x2e, - 0x04, 0x32, 0xdf, 0xcc, 0xcb, 0x13, 0xc6, 0x6e, 0x06, 0x01, 0x15, 0x51, 0x52, 0x75, 0x66, 0x0d, - 0xd9, 0xf7, 0xe1, 0xfa, 0x21, 0x62, 0xd2, 0x1b, 0x1e, 0x53, 0x9d, 0x88, 0x64, 0xb6, 0x04, 0x95, - 0x43, 0xe4, 0x0b, 0xe3, 0x2b, 0x0e, 0xff, 0xe4, 0x1b, 0xf0, 0x28, 0x41, 0xbe, 0xb0, 0xb2, 0xe2, - 0x88, 0x6f, 0xfb, 0x4f, 0x25, 0xa8, 0xa9, 0xe4, 0xcc, 0x0f, 0x98, 0x80, 0xe2, 0x33, 0x44, 0xd5, - 0xd6, 0x53, 0x90, 0xf5, 0x12, 0x74, 0xe4, 0x97, 0x4b, 0x62, 0x86, 0x49, 0x96, 0xf2, 0xdb, 0x12, - 0xfb, 0x50, 0x22, 0xc5, 0xe5, 0x9b, 0xb8, 0xfe, 0x52, 0x9d, 0xa6, 0x82, 0x38, 0xfe, 0x51, 0xc2, - 0x23, 0x5c, 0xa4, 0xf8, 0x86, 0xa3, 0x20, 0xbe, 0xd5, 0x35, 0xbf, 0x05, 0xc1, 0x4f, 0x83, 0x7c, - 0xab, 0x87, 0x24, 0x8d, 0x98, 0x1b, 0x13, 0x1c, 0x31, 0x95, 0xd3, 0x41, 0xa0, 0x86, 0x1c, 0x63, - 0xff, 0xba, 0x04, 0x8b, 0xf2, 0x02, 0x9a, 0xf7, 0xb6, 0xd9, 0xc9, 0x5a, 0xc6, 0xa2, 0x4a, 0x11, - 0xb2, 0xe4, 0x69, 0x2a, 0xbe, 0x79, 0x1c, 0x9f, 0x85, 0xf2, 0x7c, 0x50, 0xaa, 0x9d, 0x85, 0xe2, - 0x60, 0x78, 0x09, 0x3a, 0xf9, 0x01, 0x2d, 0xc6, 0xa5, 0x8a, 0xed, 0x0c, 0x2b, 0xc8, 0xe6, 0x6a, - 0x6a, 0xff, 0x98, 0xb7, 0xf4, 0xd9, 0xe5, 0xeb, 0x12, 0x54, 0xd2, 0x4c, 0x19, 0xfe, 0xc9, 0x31, - 0xa3, 0xec, 0x68, 0xe7, 0x9f, 0xd6, 0x6d, 0xe8, 0x78, 0x41, 0x80, 0xf9, 0x74, 0x6f, 0xbc, 0x8b, - 0x83, 0x2c, 0x48, 0x8b, 0x58, 0xfb, 0x6f, 0x25, 0xe8, 0x6e, 0x93, 0xf8, 0xe2, 0x43, 0x3c, 0x46, - 0x46, 0x06, 0x11, 0x4a, 0xaa, 0x93, 0x9d, 0x7f, 0xf3, 0x6a, 0xf5, 0x11, 0x1e, 0x23, 0x19, 0x5a, - 0x72, 0x65, 0xeb, 0x1c, 0x21, 0xc2, 0x4a, 0x0f, 0x66, 0xd7, 0x6e, 0x6d, 0x39, 0x78, 0x40, 0x02, - 0x51, 0x97, 0x07, 0x98, 0xba, 0xd9, 0x25, 0x5b, 0xdb, 0xa9, 0x05, 0x98, 0x8a, 0x21, 0x65, 0xc8, - 0x82, 0xb8, 0x44, 0x35, 0x0d, 0x59, 0x94, 0x18, 0x6e, 0xc8, 0x2a, 0x2c, 0x92, 0x47, 0x8f, 0x12, - 0xc4, 0x44, 0x05, 0x5d, 0x71, 0x14, 0x94, 0xa5, 0xb9, 0xba, 0x91, 0xe6, 0x56, 0xc0, 0xda, 0x45, - 0xec, 0xe1, 0xc3, 0x83, 0x9d, 0x33, 0x14, 0x31, 0x7d, 0x3a, 0xbc, 0x0e, 0x75, 0x8d, 0xfa, 0x6f, - 0xae, 0x27, 0x5f, 0x85, 0xce, 0x66, 0x10, 0x1c, 0x3e, 0xf6, 0x62, 0xed, 0x8f, 0x3e, 0xd4, 0x86, - 0xdb, 0xfb, 0x43, 0xe9, 0x92, 0x0a, 0x37, 0x40, 0x81, 0xfc, 0x34, 0xda, 0x45, 0xec, 0x00, 0x31, - 0x8a, 0xfd, 0xec, 0x34, 0xba, 0x05, 0x35, 0x85, 0xe1, 0x33, 0x43, 0xf9, 0xa9, 0xd3, 0xac, 0x02, - 0xed, 0x1f, 0x80, 0xf5, 0x23, 0x5e, 0x57, 0x21, 0x59, 0x54, 0x2b, 0x49, 0xaf, 0x42, 0xef, 0x4c, - 0x60, 0x5d, 0x59, 0x70, 0x18, 0xcb, 0xd0, 0x95, 0x03, 0x22, 0x06, 0x85, 0xec, 0x23, 0x58, 0x96, - 0x65, 0xa0, 0xe4, 0x73, 0x05, 0x16, 0xdc, 0x87, 0xd9, 0x7a, 0x56, 0x1d, 0xf1, 0x7d, 0xf7, 0x2f, - 0x3d, 0x75, 0x54, 0xa8, 0x5b, 0x07, 0x6b, 0x17, 0xba, 0x13, 0x4f, 0x44, 0x96, 0xba, 0x86, 0x9a, - 0xfd, 0x72, 0x34, 0x58, 0x5d, 0x97, 0x4f, 0x4e, 0xeb, 0xfa, 0xc9, 0x69, 0x7d, 0x27, 0x8c, 0xd9, - 0x85, 0xb5, 0x03, 0x9d, 0xe2, 0x63, 0x8a, 0xf5, 0xac, 0xae, 0xda, 0x66, 0x3c, 0xb1, 0xcc, 0x65, - 0xb3, 0x0b, 0xdd, 0x89, 0x77, 0x15, 0xad, 0xcf, 0xec, 0xe7, 0x96, 0xb9, 0x8c, 0xee, 0x43, 0xd3, - 0x78, 0x48, 0xb1, 0xfa, 0x92, 0xc9, 0xf4, 0xdb, 0xca, 0x5c, 0x06, 0xdb, 0xd0, 0x2e, 0xbc, 0x6d, - 0x58, 0x03, 0x65, 0xcf, 0x8c, 0x07, 0x8f, 0xb9, 0x4c, 0xb6, 0xa0, 0x69, 0x3c, 0x31, 0x68, 0x2d, - 0xa6, 0xdf, 0x31, 0x06, 0x37, 0x66, 0x8c, 0xa8, 0x13, 0x69, 0x17, 0xba, 0x13, 0xef, 0x0e, 0xda, - 0x25, 0xb3, 0x9f, 0x23, 0xe6, 0x2a, 0xf3, 0xb1, 0x58, 0x22, 0xa3, 0xad, 0x34, 0x96, 0x68, 0xfa, - 0x95, 0x61, 0xf0, 0xdc, 0xec, 0x41, 0xa5, 0xd5, 0x0e, 0x74, 0x8a, 0x0f, 0x0c, 0x9a, 0xd9, 0xcc, - 0x67, 0x87, 0xcb, 0xd7, 0xbb, 0xf0, 0xd6, 0x90, 0xaf, 0xf7, 0xac, 0x27, 0x88, 0xb9, 0x8c, 0x36, - 0x01, 0x54, 0x13, 0x19, 0xe0, 0x28, 0x73, 0xf4, 0x54, 0xf3, 0x9a, 0x39, 0x7a, 0x46, 0xc3, 0x79, - 0x1f, 0x40, 0xf6, 0x7e, 0x01, 0x49, 0x99, 0x75, 0x5d, 0xab, 0x31, 0xd1, 0x70, 0x0e, 0xfa, 0xd3, - 0x03, 0x53, 0x0c, 0x10, 0xa5, 0x57, 0x61, 0xf0, 0x01, 0x40, 0xde, 0x53, 0x6a, 0x06, 0x53, 0x5d, - 0xe6, 0x25, 0x3e, 0x68, 0x99, 0x1d, 0xa4, 0xa5, 0x6c, 0x9d, 0xd1, 0x55, 0x5e, 0xc2, 0xa2, 0x3b, - 0xd1, 0x21, 0x14, 0x37, 0xdb, 0x64, 0xe3, 0x30, 0x98, 0xea, 0x12, 0xac, 0x7b, 0xd0, 0x32, 0x5b, - 0x03, 0xad, 0xc5, 0x8c, 0x76, 0x61, 0x50, 0x68, 0x0f, 0xac, 0xfb, 0xd0, 0x29, 0xb6, 0x05, 0x7a, - 0x4b, 0xcd, 0x6c, 0x16, 0x06, 0xea, 0xd2, 0xcb, 0x20, 0x7f, 0x0b, 0x20, 0x6f, 0x1f, 0xb4, 0xfb, - 0xa6, 0x1a, 0x8a, 0x09, 0xa9, 0xbb, 0xd0, 0x9d, 0x68, 0x0b, 0xb4, 0xc5, 0xb3, 0xbb, 0x85, 0xb9, - 0xae, 0x7b, 0x1b, 0x20, 0x3f, 0x2e, 0xb4, 0xf4, 0xa9, 0x03, 0x64, 0xd0, 0xd6, 0x17, 0x82, 0x92, - 0x6e, 0x1b, 0xda, 0x85, 0x9e, 0x59, 0xa7, 0x99, 0x59, 0x8d, 0xf4, 0x65, 0xc9, 0xb7, 0xd8, 0x60, - 0x6a, 0xcf, 0xcd, 0x6c, 0x3b, 0x2f, 0xdb, 0x3f, 0x66, 0x57, 0xa3, 0x57, 0x6e, 0x46, 0xa7, 0xf3, - 0x2d, 0xf1, 0x6c, 0x76, 0x2e, 0x46, 0x3c, 0xcf, 0x68, 0x68, 0xe6, 0x32, 0xda, 0x83, 0xee, 0xae, - 0x2e, 0x4a, 0x55, 0xc1, 0xac, 0xd4, 0x99, 0xd1, 0x20, 0x0c, 0x06, 0xb3, 0x86, 0x54, 0x50, 0x7d, - 0x0c, 0xbd, 0xa9, 0x62, 0xd9, 0x5a, 0xcb, 0xae, 0x65, 0x67, 0x56, 0xd1, 0x73, 0xd5, 0xda, 0x87, - 0xa5, 0xc9, 0x5a, 0xd9, 0x7a, 0x5e, 0x25, 0xca, 0xd9, 0x35, 0xf4, 0x5c, 0x56, 0xef, 0x42, 0x5d, - 0xd7, 0x66, 0x96, 0xba, 0xfe, 0x9e, 0xa8, 0xd5, 0xe6, 0x4e, 0xbd, 0x07, 0x4d, 0xa3, 0x14, 0xd2, - 0xd9, 0x6e, 0xba, 0x3a, 0x1a, 0xa8, 0xdb, 0xea, 0x8c, 0xf2, 0x1e, 0xd4, 0x54, 0xf9, 0x63, 0xad, - 0x64, 0x9b, 0xdc, 0xa8, 0x86, 0x2e, 0xdb, 0x61, 0xbb, 0x88, 0x19, 0x45, 0x8d, 0x16, 0x3a, 0x5d, - 0xe7, 0xe8, 0x14, 0x5b, 0x18, 0x51, 0x6b, 0xb1, 0x09, 0x2d, 0xb3, 0xac, 0xd1, 0x4b, 0x3a, 0xa3, - 0xd4, 0x99, 0xa7, 0xc9, 0xd6, 0xf9, 0xd7, 0xdf, 0xac, 0x3d, 0xf3, 0x8f, 0x6f, 0xd6, 0x9e, 0xf9, - 0xd5, 0x93, 0xb5, 0xd2, 0xd7, 0x4f, 0xd6, 0x4a, 0x7f, 0x7f, 0xb2, 0x56, 0xfa, 0xd7, 0x93, 0xb5, - 0xd2, 0x4f, 0x7e, 0xfe, 0x3f, 0xfe, 0x0f, 0x87, 0xa6, 0x11, 0xc3, 0x21, 0xda, 0x38, 0xc3, 0x94, - 0x19, 0x43, 0xf1, 0xe9, 0x48, 0xfe, 0x19, 0xc7, 0xf8, 0x8f, 0x0e, 0xd7, 0xf2, 0x78, 0x51, 0xc0, - 0x6f, 0xfd, 0x27, 0x00, 0x00, 0xff, 0xff, 0x5c, 0x4e, 0xa6, 0xdc, 0xf0, 0x23, 0x00, 0x00, + 0xb5, 0x6c, 0x63, 0xc9, 0x5e, 0x3b, 0x58, 0x3f, 0xc2, 0x2c, 0x92, 0x56, 0x96, 0x64, 0x5b, 0xde, + 0xa1, 0x65, 0x61, 0x02, 0x02, 0x3a, 0x7a, 0xba, 0x4b, 0x33, 0x65, 0x4d, 0x77, 0xb5, 0xab, 0xab, + 0xb5, 0x1a, 0x13, 0x41, 0x70, 0x82, 0x1b, 0x47, 0x6e, 0xfc, 0x00, 0xc1, 0x1f, 0x70, 0xe1, 0xc0, + 0xc1, 0xc1, 0x89, 0x23, 0x17, 0x08, 0xbc, 0x9f, 0xc0, 0x17, 0x10, 0xf5, 0xea, 0xc7, 0x3c, 0x64, + 0x50, 0x6c, 0x04, 0x97, 0x89, 0xce, 0xac, 0xac, 0x7c, 0x55, 0x65, 0x56, 0x66, 0xd5, 0x40, 0x7f, + 0x88, 0xd9, 0x28, 0x1e, 0x6c, 0xb9, 0xc4, 0xdf, 0x3e, 0x77, 0x98, 0xf3, 0xba, 0x4b, 0x02, 0xe6, + 0xe0, 0x00, 0xd1, 0x68, 0x06, 0x8e, 0xa8, 0xbb, 0x3d, 0xc6, 0x83, 0x68, 0x3b, 0xa4, 0x84, 0x11, + 0x97, 0x8c, 0xd5, 0x57, 0xb4, 0xed, 0x0c, 0x51, 0xc0, 0xb6, 0x04, 0x60, 0x94, 0x87, 0x34, 0x74, + 0x7b, 0x35, 0xe2, 0x62, 0x89, 0xe8, 0xd5, 0xdc, 0x48, 0x7f, 0xd6, 0xd9, 0x24, 0x44, 0x91, 0x02, + 0x9e, 0x1d, 0x12, 0x32, 0x1c, 0x23, 0xc9, 0x63, 0x10, 0x9f, 0x6d, 0x23, 0x3f, 0x64, 0x13, 0x39, + 0x68, 0xfe, 0xbe, 0x08, 0xeb, 0x7b, 0x14, 0x39, 0x0c, 0xed, 0x69, 0x05, 0x2c, 0xf4, 0x65, 0x8c, + 0x22, 0x66, 0xbc, 0x00, 0x8d, 0x44, 0x29, 0x1b, 0x7b, 0xdd, 0xc2, 0xed, 0xc2, 0x66, 0xcd, 0xaa, + 0x27, 0xb8, 0x23, 0xcf, 0xb8, 0x09, 0x15, 0x74, 0x89, 0x5c, 0x3e, 0x5a, 0x14, 0xa3, 0xcb, 0x1c, + 0x3c, 0xf2, 0x8c, 0x37, 0xa1, 0x1e, 0x31, 0x8a, 0x83, 0xa1, 0x1d, 0x47, 0x88, 0x76, 0x4b, 0xb7, + 0x0b, 0x9b, 0xf5, 0x7b, 0x2b, 0x5b, 0x5c, 0xe5, 0xad, 0x13, 0x31, 0x70, 0x1a, 0x21, 0x6a, 0x41, + 0x94, 0x7c, 0x1b, 0x77, 0xa1, 0xe2, 0xa1, 0x0b, 0xec, 0xa2, 0xa8, 0x5b, 0xbe, 0x5d, 0xda, 0xac, + 0xdf, 0x6b, 0x48, 0xf2, 0x87, 0x02, 0x69, 0xe9, 0x41, 0xe3, 0x15, 0xa8, 0x46, 0x8c, 0x50, 0x67, + 0x88, 0xa2, 0xee, 0x92, 0x20, 0x6c, 0x6a, 0xbe, 0x02, 0x6b, 0x25, 0xc3, 0xc6, 0x73, 0x50, 0x7a, + 0xb4, 0x77, 0xd4, 0x5d, 0x16, 0xd2, 0x41, 0x51, 0x85, 0xc8, 0xb5, 0x38, 0xda, 0xb8, 0x03, 0xcd, + 0xc8, 0x09, 0xbc, 0x01, 0xb9, 0xb4, 0x43, 0xec, 0x05, 0x51, 0xb7, 0x72, 0xbb, 0xb0, 0x59, 0xb5, + 0x1a, 0x0a, 0xd9, 0xe7, 0x38, 0xf3, 0x3d, 0xb8, 0x71, 0xc2, 0x1c, 0xca, 0xae, 0xe1, 0x1d, 0xf3, + 0x14, 0xd6, 0x2d, 0xe4, 0x93, 0x8b, 0x6b, 0xb9, 0xb6, 0x0b, 0x15, 0x86, 0x7d, 0x44, 0x62, 0x26, + 0x5c, 0xdb, 0xb4, 0x34, 0x68, 0xfe, 0xb1, 0x00, 0xc6, 0xfe, 0x25, 0x72, 0xfb, 0x94, 0xb8, 0x28, + 0x8a, 0xfe, 0x4f, 0xcb, 0xf5, 0x32, 0x54, 0x42, 0xa9, 0x40, 0xb7, 0x2c, 0xc8, 0xd5, 0x2a, 0x68, + 0xad, 0xf4, 0xa8, 0xf9, 0x05, 0xac, 0x9d, 0xe0, 0x61, 0xe0, 0x8c, 0x9f, 0xa2, 0xbe, 0xeb, 0xb0, + 0x1c, 0x09, 0x9e, 0x42, 0xd5, 0xa6, 0xa5, 0x20, 0xb3, 0x0f, 0xc6, 0xe7, 0x0e, 0x66, 0x4f, 0x4f, + 0x92, 0xf9, 0x3a, 0xac, 0xe6, 0x38, 0x46, 0x21, 0x09, 0x22, 0x24, 0x14, 0x60, 0x0e, 0x8b, 0x23, + 0xc1, 0x6c, 0xc9, 0x52, 0x90, 0x49, 0x60, 0xfd, 0x34, 0xf4, 0xae, 0x19, 0x4d, 0xf7, 0xa0, 0x46, + 0x51, 0x44, 0x62, 0xca, 0x63, 0xa0, 0x28, 0x9c, 0xba, 0x26, 0x9d, 0xfa, 0x09, 0x0e, 0xe2, 0x4b, + 0x4b, 0x8f, 0x59, 0x29, 0x99, 0xda, 0x9f, 0x2c, 0xba, 0xce, 0xfe, 0x7c, 0x0f, 0x6e, 0xf4, 0x9d, + 0x38, 0xba, 0x8e, 0xae, 0xe6, 0xfb, 0x7c, 0x6f, 0x47, 0xb1, 0x7f, 0xad, 0xc9, 0x7f, 0x28, 0x40, + 0x75, 0x2f, 0x8c, 0x4f, 0x23, 0x67, 0x88, 0x8c, 0xef, 0x40, 0x9d, 0x11, 0xe6, 0x8c, 0xed, 0x98, + 0x83, 0x82, 0xbc, 0x6c, 0x81, 0x40, 0x49, 0x82, 0x17, 0xa0, 0x11, 0x22, 0xea, 0x86, 0xb1, 0xa2, + 0x28, 0xde, 0x2e, 0x6d, 0x96, 0xad, 0xba, 0xc4, 0x49, 0x92, 0x2d, 0x58, 0x15, 0x63, 0x36, 0x0e, + 0xec, 0x73, 0x44, 0x03, 0x34, 0xf6, 0x89, 0x87, 0xc4, 0xe6, 0x28, 0x5b, 0x1d, 0x31, 0x74, 0x14, + 0x7c, 0x9c, 0x0c, 0x18, 0xaf, 0x42, 0x27, 0xa1, 0xe7, 0x3b, 0x5e, 0x50, 0x97, 0x05, 0x75, 0x5b, + 0x51, 0x9f, 0x2a, 0xb4, 0xf9, 0x4b, 0x68, 0x7d, 0x36, 0xa2, 0x84, 0xb1, 0x31, 0x0e, 0x86, 0x0f, + 0x1d, 0xe6, 0xf0, 0xd0, 0x0c, 0x11, 0xc5, 0xc4, 0x8b, 0x94, 0xb6, 0x1a, 0x34, 0x5e, 0x83, 0x0e, + 0x93, 0xb4, 0xc8, 0xb3, 0x35, 0x4d, 0x51, 0xd0, 0xac, 0x24, 0x03, 0x7d, 0x45, 0xfc, 0x12, 0xb4, + 0x52, 0x62, 0x1e, 0xdc, 0x4a, 0xdf, 0x66, 0x82, 0xfd, 0x0c, 0xfb, 0xc8, 0xbc, 0x10, 0xbe, 0x12, + 0x8b, 0x6c, 0xbc, 0x06, 0xb5, 0xd4, 0x0f, 0x05, 0xb1, 0x43, 0x5a, 0x72, 0x87, 0x68, 0x77, 0x5a, + 0xd5, 0xc4, 0x29, 0x1f, 0x40, 0x9b, 0x25, 0x8a, 0xdb, 0x9e, 0xc3, 0x9c, 0xfc, 0xa6, 0xca, 0x5b, + 0x65, 0xb5, 0x58, 0x0e, 0x36, 0xdf, 0x87, 0x5a, 0x1f, 0x7b, 0x91, 0x14, 0xdc, 0x85, 0x8a, 0x1b, + 0x53, 0x8a, 0x02, 0xa6, 0x4d, 0x56, 0xa0, 0xb1, 0x06, 0x4b, 0x63, 0xec, 0x63, 0xa6, 0xcc, 0x94, + 0x80, 0x49, 0x00, 0x8e, 0x91, 0x4f, 0xe8, 0x44, 0x38, 0x6c, 0x0d, 0x96, 0xb2, 0x8b, 0x2b, 0x01, + 0xe3, 0x59, 0xa8, 0xf9, 0xce, 0x65, 0xb2, 0xa8, 0x7c, 0xa4, 0xea, 0x3b, 0x97, 0x52, 0xf9, 0x2e, + 0x54, 0xce, 0x1c, 0x3c, 0x76, 0x03, 0xa6, 0xbc, 0xa2, 0xc1, 0x54, 0x60, 0x39, 0x2b, 0xf0, 0x2f, + 0x45, 0xa8, 0x4b, 0x89, 0x52, 0xe1, 0x35, 0x58, 0x72, 0x1d, 0x77, 0x94, 0x88, 0x14, 0x80, 0x71, + 0x57, 0x2b, 0x52, 0xcc, 0x66, 0xb8, 0x54, 0x53, 0xad, 0xda, 0x36, 0x40, 0xf4, 0xd8, 0x09, 0x95, + 0x6e, 0xa5, 0x05, 0xc4, 0x35, 0x4e, 0x23, 0xd5, 0x7d, 0x0b, 0x1a, 0x72, 0xdf, 0xa9, 0x29, 0xe5, + 0x05, 0x53, 0xea, 0x92, 0x4a, 0x4e, 0xba, 0x03, 0xcd, 0x38, 0x42, 0xf6, 0x08, 0x23, 0xea, 0x50, + 0x77, 0x34, 0xe9, 0x2e, 0xc9, 0x03, 0x28, 0x8e, 0xd0, 0xa1, 0xc6, 0x19, 0xf7, 0x60, 0x89, 0xe7, + 0x96, 0xa8, 0xbb, 0x2c, 0xce, 0xba, 0xe7, 0xb2, 0x2c, 0x85, 0xa9, 0x5b, 0xe2, 0x77, 0x3f, 0x60, + 0x74, 0x62, 0x49, 0xd2, 0xde, 0x3b, 0x00, 0x29, 0xd2, 0x58, 0x81, 0xd2, 0x39, 0x9a, 0xa8, 0x38, + 0xe4, 0x9f, 0xdc, 0x39, 0x17, 0xce, 0x38, 0xd6, 0x5e, 0x97, 0xc0, 0x7b, 0xc5, 0x77, 0x0a, 0xa6, + 0x0b, 0xed, 0xdd, 0xf1, 0x39, 0x26, 0x99, 0xe9, 0x6b, 0xb0, 0xe4, 0x3b, 0x5f, 0x10, 0xaa, 0x3d, + 0x29, 0x00, 0x81, 0xc5, 0x01, 0xa1, 0x9a, 0x85, 0x00, 0x8c, 0x16, 0x14, 0x49, 0x28, 0xfc, 0x55, + 0xb3, 0x8a, 0x24, 0x4c, 0x05, 0x95, 0x33, 0x82, 0xcc, 0x7f, 0x96, 0x01, 0x52, 0x29, 0x86, 0x05, + 0x3d, 0x4c, 0xec, 0x08, 0x51, 0x7e, 0xbe, 0xdb, 0x83, 0x09, 0x43, 0x91, 0x4d, 0x91, 0x1b, 0xd3, + 0x08, 0x5f, 0xf0, 0xf5, 0xe3, 0x66, 0xdf, 0x90, 0x66, 0x4f, 0xe9, 0x66, 0xdd, 0xc4, 0xe4, 0x44, + 0xce, 0xdb, 0xe5, 0xd3, 0x2c, 0x3d, 0xcb, 0x38, 0x82, 0x1b, 0x29, 0x4f, 0x2f, 0xc3, 0xae, 0x78, + 0x15, 0xbb, 0xd5, 0x84, 0x9d, 0x97, 0xb2, 0xda, 0x87, 0x55, 0x4c, 0xec, 0x2f, 0x63, 0x14, 0xe7, + 0x18, 0x95, 0xae, 0x62, 0xd4, 0xc1, 0xe4, 0x87, 0x62, 0x42, 0xca, 0xa6, 0x0f, 0xb7, 0x32, 0x56, + 0xf2, 0x70, 0xcf, 0x30, 0x2b, 0x5f, 0xc5, 0x6c, 0x3d, 0xd1, 0x8a, 0xe7, 0x83, 0x94, 0xe3, 0x47, + 0xb0, 0x8e, 0x89, 0xfd, 0xd8, 0xc1, 0x6c, 0x9a, 0xdd, 0xd2, 0xb7, 0x18, 0xc9, 0x4f, 0xb4, 0x3c, + 0x2f, 0x69, 0xa4, 0x8f, 0xe8, 0x30, 0x67, 0xe4, 0xf2, 0xb7, 0x18, 0x79, 0x2c, 0x26, 0xa4, 0x6c, + 0x76, 0xa0, 0x83, 0xc9, 0xb4, 0x36, 0x95, 0xab, 0x98, 0xb4, 0x31, 0xc9, 0x6b, 0xb2, 0x0b, 0x9d, + 0x08, 0xb9, 0x8c, 0xd0, 0xec, 0x26, 0xa8, 0x5e, 0xc5, 0x62, 0x45, 0xd1, 0x27, 0x3c, 0xcc, 0x9f, + 0x42, 0xe3, 0x30, 0x1e, 0x22, 0x36, 0x1e, 0x24, 0xc9, 0xe0, 0xa9, 0xe5, 0x1f, 0xf3, 0xdf, 0x45, + 0xa8, 0xef, 0x0d, 0x29, 0x89, 0xc3, 0x5c, 0x4e, 0x96, 0x41, 0x3a, 0x9d, 0x93, 0x05, 0x89, 0xc8, + 0xc9, 0x92, 0xf8, 0x6d, 0x68, 0xf8, 0x22, 0x74, 0x15, 0xbd, 0xcc, 0x43, 0x9d, 0x99, 0xa0, 0xb6, + 0xea, 0x7e, 0x26, 0x99, 0x6d, 0x01, 0x84, 0xd8, 0x8b, 0xd4, 0x1c, 0x99, 0x8e, 0xda, 0xaa, 0xdc, + 0xd2, 0x29, 0xda, 0xaa, 0x85, 0x49, 0xb6, 0x7e, 0x13, 0xea, 0x03, 0xee, 0x24, 0x35, 0x21, 0x97, + 0x8c, 0x52, 0xef, 0x59, 0x30, 0x48, 0x83, 0xf0, 0x10, 0x9a, 0x23, 0xe9, 0x32, 0x35, 0x49, 0xee, + 0xa1, 0x3b, 0xca, 0x92, 0xd4, 0xde, 0xad, 0xac, 0x67, 0xe5, 0x02, 0x34, 0x46, 0x19, 0x54, 0xef, + 0x04, 0x3a, 0x33, 0x24, 0x73, 0x72, 0xd0, 0x66, 0x36, 0x07, 0xd5, 0xef, 0x19, 0x52, 0x50, 0x76, + 0x66, 0x36, 0x2f, 0xfd, 0xb6, 0x08, 0x8d, 0x4f, 0x11, 0x7b, 0x4c, 0xe8, 0xb9, 0xd4, 0xd7, 0x80, + 0x72, 0xe0, 0xf8, 0x48, 0x71, 0x14, 0xdf, 0xc6, 0x2d, 0xa8, 0xd2, 0x4b, 0x99, 0x40, 0xd4, 0x7a, + 0x56, 0xe8, 0xa5, 0x48, 0x0c, 0xc6, 0xf3, 0x00, 0xf4, 0xd2, 0x0e, 0x1d, 0xf7, 0x1c, 0x29, 0x0f, + 0x96, 0xad, 0x1a, 0xbd, 0xec, 0x4b, 0x04, 0xdf, 0x0a, 0xf4, 0xd2, 0x46, 0x94, 0x12, 0x1a, 0xa9, + 0x5c, 0x55, 0xa5, 0x97, 0xfb, 0x02, 0x56, 0x73, 0x3d, 0x4a, 0xc2, 0x10, 0x79, 0x22, 0x47, 0x8b, + 0xb9, 0x0f, 0x25, 0x82, 0x4b, 0x65, 0x5a, 0xea, 0xb2, 0x94, 0xca, 0x52, 0xa9, 0x2c, 0x95, 0x5a, + 0x91, 0x33, 0x59, 0x56, 0x2a, 0x4b, 0xa4, 0x56, 0xa5, 0x54, 0x96, 0x91, 0xca, 0x52, 0xa9, 0x35, + 0x3d, 0x57, 0x49, 0x35, 0x7f, 0x53, 0x80, 0xf5, 0xe9, 0xc2, 0x4f, 0xd5, 0xa6, 0x6f, 0x43, 0xc3, + 0x15, 0xeb, 0x95, 0xdb, 0x93, 0x9d, 0x99, 0x95, 0xb4, 0xea, 0x6e, 0x66, 0x1b, 0xdf, 0x87, 0x66, + 0x20, 0x1d, 0x9c, 0x6c, 0xcd, 0x52, 0xba, 0x2e, 0x59, 0xdf, 0x5b, 0x8d, 0x20, 0x03, 0x99, 0x1e, + 0x18, 0x9f, 0x53, 0xcc, 0xd0, 0x09, 0xa3, 0xc8, 0xf1, 0x9f, 0x46, 0x75, 0x6f, 0x40, 0x59, 0x54, + 0x2b, 0x7c, 0x99, 0x1a, 0x96, 0xf8, 0x36, 0x5f, 0x86, 0xd5, 0x9c, 0x14, 0x65, 0xeb, 0x0a, 0x94, + 0xc6, 0x28, 0x10, 0xdc, 0x9b, 0x16, 0xff, 0x34, 0x1d, 0xe8, 0x58, 0xc8, 0xf1, 0x9e, 0x9e, 0x36, + 0x4a, 0x44, 0x29, 0x15, 0xb1, 0x09, 0x46, 0x56, 0x84, 0x52, 0x45, 0x6b, 0x5d, 0xc8, 0x68, 0xfd, + 0x08, 0x3a, 0x7b, 0x63, 0x12, 0xa1, 0x13, 0xe6, 0xe1, 0xe0, 0x69, 0xb4, 0x23, 0xbf, 0x80, 0xd5, + 0xcf, 0xd8, 0xe4, 0x73, 0xce, 0x2c, 0xc2, 0x5f, 0xa1, 0xa7, 0x64, 0x1f, 0x25, 0x8f, 0xb5, 0x7d, + 0x94, 0x3c, 0xe6, 0xcd, 0x8d, 0x4b, 0xc6, 0xb1, 0x1f, 0x88, 0x50, 0x68, 0x5a, 0x0a, 0x32, 0x77, + 0xa1, 0x21, 0x6b, 0xe8, 0x63, 0xe2, 0xc5, 0x63, 0x34, 0x37, 0x06, 0x37, 0x00, 0x42, 0x87, 0x3a, + 0x3e, 0x62, 0x88, 0xca, 0x3d, 0x54, 0xb3, 0x32, 0x18, 0xf3, 0x77, 0x45, 0x58, 0x93, 0xf7, 0x0d, + 0x27, 0xb2, 0xcd, 0xd6, 0x26, 0xf4, 0xa0, 0x3a, 0x22, 0x11, 0xcb, 0x30, 0x4c, 0x60, 0xae, 0x22, + 0xef, 0xcf, 0x25, 0x37, 0xfe, 0x99, 0xbb, 0x04, 0x28, 0x5d, 0x7d, 0x09, 0x30, 0xd3, 0xe6, 0x97, + 0x67, 0xdb, 0x7c, 0x1e, 0x6d, 0x9a, 0x08, 0xcb, 0x18, 0xaf, 0x59, 0x35, 0x85, 0x39, 0xf2, 0x8c, + 0xbb, 0xd0, 0x1e, 0x72, 0x2d, 0xed, 0x11, 0x21, 0xe7, 0x76, 0xe8, 0xb0, 0x91, 0x08, 0xf5, 0x9a, + 0xd5, 0x14, 0xe8, 0x43, 0x42, 0xce, 0xfb, 0x0e, 0x1b, 0x19, 0xef, 0x42, 0x4b, 0x95, 0x81, 0xbe, + 0x70, 0x51, 0xa4, 0x0e, 0x3f, 0x15, 0x45, 0x59, 0xef, 0x59, 0xcd, 0xf3, 0x0c, 0x14, 0x99, 0x37, + 0xe1, 0xc6, 0x43, 0x14, 0x31, 0x4a, 0x26, 0x79, 0xc7, 0x98, 0xdf, 0x07, 0x38, 0x0a, 0x18, 0xa2, + 0x67, 0x8e, 0x8b, 0x22, 0xe3, 0x8d, 0x2c, 0xa4, 0x8a, 0xa3, 0x95, 0x2d, 0x79, 0xdd, 0x93, 0x0c, + 0x58, 0x19, 0x1a, 0x73, 0x0b, 0x96, 0x2d, 0x12, 0xf3, 0x74, 0xf4, 0xa2, 0xfe, 0x52, 0xf3, 0x1a, + 0x6a, 0x9e, 0x40, 0x5a, 0x6a, 0xcc, 0x3c, 0xd4, 0x2d, 0x6c, 0xca, 0x4e, 0x2d, 0xd1, 0x16, 0xd4, + 0xb0, 0xc6, 0xa9, 0xac, 0x32, 0x2b, 0x3a, 0x25, 0x31, 0xdf, 0x87, 0x55, 0xc9, 0x49, 0x72, 0xd6, + 0x6c, 0x5e, 0x84, 0x65, 0xaa, 0xd5, 0x28, 0xa4, 0xf7, 0x3c, 0x8a, 0x48, 0x8d, 0x71, 0x7f, 0x7c, + 0x82, 0x23, 0x96, 0x1a, 0xa2, 0xfd, 0xb1, 0x0a, 0x1d, 0x3e, 0x90, 0xe3, 0x69, 0x7e, 0x08, 0x8d, + 0x1d, 0xab, 0xff, 0x29, 0xc2, 0xc3, 0xd1, 0x80, 0x67, 0xcf, 0xef, 0xe5, 0x61, 0x65, 0xb0, 0xa1, + 0xb4, 0xcd, 0x0c, 0x59, 0x39, 0x3a, 0xf3, 0x23, 0x58, 0xdf, 0xf1, 0xbc, 0x2c, 0x4a, 0x6b, 0xfd, + 0x06, 0xd4, 0x82, 0x0c, 0xbb, 0xcc, 0x99, 0x95, 0xa3, 0x4e, 0x89, 0xcc, 0x9f, 0xc1, 0xea, 0xa3, + 0x60, 0x8c, 0x03, 0xb4, 0xd7, 0x3f, 0x3d, 0x46, 0x49, 0x2e, 0x32, 0xa0, 0xcc, 0x6b, 0x36, 0xc1, + 0xa3, 0x6a, 0x89, 0x6f, 0x1e, 0x9c, 0xc1, 0xc0, 0x76, 0xc3, 0x38, 0x52, 0x97, 0x3d, 0xcb, 0xc1, + 0x60, 0x2f, 0x8c, 0x23, 0x7e, 0xb8, 0xf0, 0xe2, 0x82, 0x04, 0xe3, 0x89, 0x88, 0xd0, 0xaa, 0x55, + 0x71, 0xc3, 0xf8, 0x51, 0x30, 0x9e, 0x98, 0xdf, 0x15, 0x1d, 0x38, 0x42, 0x9e, 0xe5, 0x04, 0x1e, + 0xf1, 0x1f, 0xa2, 0x8b, 0x8c, 0x84, 0xa4, 0xdb, 0xd3, 0x99, 0xe8, 0xeb, 0x02, 0x34, 0x76, 0x86, + 0x28, 0x60, 0x0f, 0x11, 0x73, 0xf0, 0x58, 0x74, 0x74, 0x17, 0x88, 0x46, 0x98, 0x04, 0x2a, 0xdc, + 0x34, 0xc8, 0x1b, 0x72, 0x1c, 0x60, 0x66, 0x7b, 0x0e, 0xf2, 0x49, 0x20, 0xb8, 0x54, 0x2d, 0xe0, + 0xa8, 0x87, 0x02, 0x63, 0xbc, 0x0c, 0x6d, 0x79, 0x19, 0x67, 0x8f, 0x9c, 0xc0, 0x1b, 0xf3, 0x40, + 0x2f, 0x89, 0xd0, 0x6c, 0x49, 0xf4, 0xa1, 0xc2, 0x1a, 0xaf, 0xc0, 0x8a, 0x0a, 0xc3, 0x94, 0xb2, + 0x2c, 0x28, 0xdb, 0x0a, 0x9f, 0x23, 0x8d, 0xc3, 0x90, 0x50, 0x16, 0xd9, 0x11, 0x72, 0x5d, 0xe2, + 0x87, 0xaa, 0x1d, 0x6a, 0x6b, 0xfc, 0x89, 0x44, 0x9b, 0x43, 0x58, 0x3d, 0xe0, 0x76, 0x2a, 0x4b, + 0xd2, 0x6d, 0xd5, 0xf2, 0x91, 0x6f, 0x0f, 0xc6, 0xc4, 0x3d, 0xb7, 0x79, 0x72, 0x54, 0x1e, 0xe6, + 0x05, 0xd7, 0x2e, 0x47, 0x9e, 0xe0, 0xaf, 0x44, 0xe7, 0xcf, 0xa9, 0x46, 0x84, 0x85, 0xe3, 0x78, + 0x68, 0x87, 0x94, 0x0c, 0x90, 0x32, 0xb1, 0xed, 0x23, 0xff, 0x50, 0xe2, 0xfb, 0x1c, 0x6d, 0xfe, + 0xa9, 0x00, 0x6b, 0x79, 0x49, 0x2a, 0xd5, 0x6f, 0xc3, 0x5a, 0x5e, 0x94, 0x3a, 0xfe, 0x65, 0x79, + 0xd9, 0xc9, 0x0a, 0x94, 0x85, 0xc0, 0x7d, 0x68, 0x8a, 0xab, 0x5b, 0xdb, 0x93, 0x9c, 0xf2, 0x45, + 0x4f, 0x76, 0x5d, 0xac, 0x86, 0x93, 0x5d, 0xa5, 0x77, 0xe1, 0x96, 0x32, 0xdf, 0x9e, 0x55, 0x5b, + 0x6e, 0x88, 0x75, 0x45, 0x70, 0x3c, 0xa5, 0xfd, 0x27, 0xd0, 0x4d, 0x51, 0xbb, 0x13, 0x81, 0x4c, + 0x37, 0xf3, 0xea, 0x94, 0xb1, 0x3b, 0x9e, 0x47, 0x45, 0x94, 0x94, 0xad, 0x79, 0x43, 0xe6, 0x03, + 0xb8, 0x79, 0x82, 0x98, 0xf4, 0x86, 0xc3, 0x54, 0x27, 0x22, 0x99, 0xad, 0x40, 0xe9, 0x04, 0xb9, + 0xc2, 0xf8, 0x92, 0xc5, 0x3f, 0xf9, 0x06, 0x3c, 0x8d, 0x90, 0x2b, 0xac, 0x2c, 0x59, 0xe2, 0xdb, + 0x0c, 0xa1, 0xf2, 0xe1, 0xc9, 0x01, 0xaf, 0x37, 0xf8, 0xa6, 0x96, 0xf5, 0x89, 0x3a, 0x8b, 0x9a, + 0x56, 0x45, 0xc0, 0x47, 0x9e, 0xf1, 0x11, 0xac, 0xca, 0x21, 0x77, 0xe4, 0x04, 0x43, 0x64, 0x87, + 0x64, 0x8c, 0x5d, 0xb9, 0xf5, 0x5b, 0xf7, 0x7a, 0x2a, 0x7c, 0x15, 0x9f, 0x3d, 0x41, 0xd2, 0x17, + 0x14, 0x56, 0x67, 0x38, 0x8d, 0x32, 0xff, 0x51, 0x80, 0x8a, 0x3a, 0x0e, 0xf8, 0x91, 0xe6, 0x51, + 0x7c, 0x81, 0xa8, 0xda, 0xec, 0x0a, 0x32, 0x5e, 0x82, 0x96, 0xfc, 0xb2, 0x49, 0xc8, 0x30, 0x49, + 0x0e, 0x99, 0xa6, 0xc4, 0x3e, 0x92, 0x48, 0x71, 0xdd, 0x27, 0x2e, 0xdc, 0x54, 0x6f, 0xab, 0x20, + 0x8e, 0x3f, 0x8b, 0xb8, 0x52, 0xe2, 0x50, 0xa9, 0x59, 0x0a, 0xe2, 0xc1, 0xa5, 0xf9, 0x2d, 0x09, + 0x7e, 0x1a, 0xe4, 0xc1, 0xe5, 0x93, 0x38, 0x60, 0x76, 0x48, 0x70, 0xc0, 0xd4, 0x29, 0x02, 0x02, + 0xd5, 0xe7, 0x18, 0x63, 0x13, 0xaa, 0x67, 0x91, 0x2d, 0xac, 0x11, 0x15, 0x63, 0x72, 0xb2, 0x29, + 0xab, 0xad, 0xca, 0x59, 0x24, 0x3e, 0xcc, 0x5f, 0x17, 0x60, 0x59, 0x5e, 0x8e, 0xf3, 0xbe, 0x3b, + 0x39, 0xf5, 0x8b, 0x58, 0x54, 0x50, 0x42, 0x2b, 0x79, 0xd2, 0x8b, 0x6f, 0x9e, 0x63, 0x2e, 0x7c, + 0x79, 0x76, 0x29, 0x23, 0x2e, 0x7c, 0x71, 0x68, 0xbd, 0x04, 0xad, 0xb4, 0x78, 0x10, 0xe3, 0xd2, + 0x98, 0x66, 0x82, 0x15, 0x64, 0x0b, 0x6d, 0x32, 0x7f, 0x0c, 0x90, 0x5e, 0x12, 0xf3, 0xed, 0x10, + 0x27, 0xca, 0xf0, 0x4f, 0x8e, 0x19, 0x26, 0x65, 0x07, 0xff, 0x34, 0xee, 0x42, 0xcb, 0xf1, 0x3c, + 0xcc, 0xa7, 0x3b, 0xe3, 0x03, 0xec, 0x25, 0x09, 0x24, 0x8f, 0x35, 0xff, 0x5a, 0x80, 0xf6, 0x1e, + 0x09, 0x27, 0x1f, 0xe2, 0x31, 0xca, 0x64, 0x37, 0xa1, 0xa4, 0xaa, 0x3a, 0xf8, 0x37, 0xaf, 0xa4, + 0xcf, 0xf0, 0x18, 0xc9, 0xb0, 0x97, 0xbb, 0xae, 0xca, 0x11, 0x22, 0xe4, 0xf5, 0x60, 0x72, 0x25, + 0xd8, 0x94, 0x83, 0xc7, 0xc4, 0x13, 0x3d, 0x83, 0x87, 0xa9, 0x9d, 0x5c, 0x00, 0x36, 0xad, 0x8a, + 0x87, 0xa9, 0x18, 0x52, 0x86, 0x2c, 0x89, 0x0b, 0xde, 0xac, 0x21, 0xcb, 0x12, 0xc3, 0x0d, 0x59, + 0x87, 0x65, 0x72, 0x76, 0x16, 0x21, 0x26, 0xd6, 0xaa, 0x64, 0x29, 0x28, 0x49, 0xc1, 0xd5, 0x4c, + 0x0a, 0x5e, 0x03, 0xe3, 0x00, 0xb1, 0x47, 0x8f, 0x8e, 0xf7, 0x2f, 0x50, 0xc0, 0xf4, 0xc9, 0xf5, + 0x3a, 0x54, 0x35, 0xea, 0xbf, 0xb9, 0x3a, 0x7d, 0x15, 0x5a, 0x3b, 0x9e, 0x77, 0xf2, 0xd8, 0x09, + 0xb5, 0x3f, 0xba, 0x50, 0xe9, 0xef, 0x1d, 0xf5, 0xa5, 0x4b, 0x4a, 0xdc, 0x00, 0x05, 0xf2, 0x93, + 0xf2, 0x00, 0xb1, 0x63, 0xc4, 0x28, 0x76, 0x93, 0x93, 0xf2, 0x0e, 0x54, 0x14, 0x86, 0xcf, 0xf4, + 0xe5, 0xa7, 0x3e, 0x02, 0x14, 0x68, 0xfe, 0x00, 0x8c, 0x1f, 0xf1, 0x9a, 0x0f, 0xc9, 0x82, 0x5f, + 0x49, 0x7a, 0x15, 0x3a, 0x17, 0x02, 0x6b, 0xcb, 0x62, 0x28, 0xb3, 0x0c, 0x6d, 0x39, 0x20, 0xf2, + 0x83, 0x90, 0x7d, 0x0a, 0xab, 0xb2, 0x44, 0x95, 0x7c, 0xae, 0xc1, 0x82, 0xfb, 0x30, 0x59, 0xcf, + 0xb2, 0x25, 0xbe, 0xef, 0xfd, 0xb9, 0xa3, 0x8e, 0x31, 0x75, 0x23, 0x62, 0x1c, 0x40, 0x7b, 0xea, + 0xf9, 0xca, 0x50, 0x57, 0x64, 0xf3, 0x5f, 0xb5, 0x7a, 0xeb, 0x5b, 0xf2, 0x39, 0x6c, 0x4b, 0x3f, + 0x87, 0x6d, 0xed, 0xfb, 0x21, 0x9b, 0x18, 0xfb, 0xd0, 0xca, 0x3f, 0xf4, 0x18, 0xcf, 0xea, 0x8a, + 0x72, 0xce, 0xf3, 0xcf, 0x42, 0x36, 0x07, 0xd0, 0x9e, 0x7a, 0xf3, 0xd1, 0xfa, 0xcc, 0x7f, 0x0a, + 0x5a, 0xc8, 0xe8, 0x01, 0xd4, 0x33, 0x8f, 0x3c, 0x46, 0x57, 0x32, 0x99, 0x7d, 0xf7, 0x59, 0xc8, + 0x60, 0x0f, 0x9a, 0xb9, 0x77, 0x17, 0xa3, 0xa7, 0xec, 0x99, 0xf3, 0x18, 0xb3, 0x90, 0xc9, 0x2e, + 0xd4, 0x33, 0xcf, 0x1f, 0x5a, 0x8b, 0xd9, 0x37, 0x96, 0xde, 0xad, 0x39, 0x23, 0xea, 0xb4, 0x3c, + 0x80, 0xf6, 0xd4, 0x9b, 0x88, 0x76, 0xc9, 0xfc, 0xa7, 0x92, 0x85, 0xca, 0x7c, 0x2c, 0x96, 0x28, + 0xd3, 0xf2, 0x66, 0x96, 0x68, 0xf6, 0x05, 0xa4, 0xf7, 0xdc, 0xfc, 0x41, 0xa5, 0xd5, 0x3e, 0xb4, + 0xf2, 0x8f, 0x1f, 0x9a, 0xd9, 0xdc, 0x27, 0x91, 0xab, 0xd7, 0x3b, 0xf7, 0x0e, 0x92, 0xae, 0xf7, + 0xbc, 0xe7, 0x91, 0x85, 0x8c, 0x76, 0x00, 0x54, 0x83, 0xeb, 0xe1, 0x20, 0x71, 0xf4, 0x4c, 0x63, + 0x9d, 0x38, 0x7a, 0x4e, 0x33, 0xfc, 0x00, 0x40, 0xf6, 0xa5, 0x1e, 0x89, 0x99, 0x71, 0x53, 0xab, + 0x31, 0xd5, 0x0c, 0xf7, 0xba, 0xb3, 0x03, 0x33, 0x0c, 0x10, 0xa5, 0xd7, 0x61, 0xf0, 0x01, 0x40, + 0xda, 0xef, 0x6a, 0x06, 0x33, 0x1d, 0xf0, 0x15, 0x3e, 0x68, 0x64, 0xbb, 0x5b, 0x43, 0xd9, 0x3a, + 0xa7, 0xe3, 0xbd, 0x82, 0x45, 0x7b, 0xaa, 0x7b, 0xc9, 0x6f, 0xb6, 0xe9, 0xa6, 0xa6, 0x37, 0xd3, + 0xc1, 0x18, 0xf7, 0xa1, 0x91, 0x6d, 0x5b, 0xb4, 0x16, 0x73, 0x5a, 0x99, 0x5e, 0xae, 0x75, 0x31, + 0x1e, 0x40, 0x2b, 0xdf, 0xb2, 0xe8, 0x2d, 0x35, 0xb7, 0x91, 0xe9, 0xa9, 0x0b, 0xb9, 0x0c, 0xf9, + 0x5b, 0x00, 0x69, 0x6b, 0xa3, 0xdd, 0x37, 0xd3, 0xec, 0x4c, 0x49, 0x3d, 0x80, 0xf6, 0x54, 0xcb, + 0xa2, 0x2d, 0x9e, 0xdf, 0xc9, 0x2c, 0x74, 0xdd, 0xdb, 0x00, 0xe9, 0x71, 0xa1, 0xa5, 0xcf, 0x1c, + 0x20, 0xbd, 0xa6, 0xbe, 0xac, 0x94, 0x74, 0x7b, 0xd0, 0xcc, 0xf5, 0xf3, 0x3a, 0xcd, 0xcc, 0x6b, + 0xf2, 0xaf, 0x4a, 0xbe, 0xf9, 0xe6, 0x57, 0x7b, 0x6e, 0x6e, 0x4b, 0x7c, 0xd5, 0xfe, 0xc9, 0x76, + 0x5c, 0x7a, 0xe5, 0xe6, 0x74, 0x61, 0xdf, 0x12, 0xcf, 0xd9, 0xae, 0x2a, 0x13, 0xcf, 0x73, 0x9a, + 0xad, 0x85, 0x8c, 0x0e, 0xa1, 0x7d, 0xa0, 0x0b, 0x66, 0x55, 0xcc, 0x2b, 0x75, 0xe6, 0x34, 0x2f, + 0xbd, 0xde, 0xbc, 0x21, 0x15, 0x54, 0x1f, 0x43, 0x67, 0xa6, 0x90, 0x37, 0x36, 0x92, 0x2b, 0xe3, + 0xb9, 0x15, 0xfe, 0x42, 0xb5, 0x8e, 0x60, 0x65, 0xba, 0x8e, 0x37, 0x9e, 0x57, 0x89, 0x72, 0x7e, + 0x7d, 0xbf, 0x90, 0xd5, 0xbb, 0x50, 0xd5, 0xb5, 0x99, 0xa1, 0xae, 0xe6, 0xa7, 0x6a, 0xb5, 0x85, + 0x53, 0xef, 0x43, 0x3d, 0x53, 0x0a, 0xe9, 0x6c, 0x37, 0x5b, 0x1d, 0xf5, 0xd4, 0x4d, 0x7a, 0x42, + 0x79, 0x1f, 0x2a, 0xaa, 0xfc, 0x31, 0xd6, 0x92, 0x4d, 0x9e, 0xa9, 0x86, 0xae, 0xda, 0x61, 0x07, + 0x88, 0x65, 0x8a, 0x1a, 0x2d, 0x74, 0xb6, 0xce, 0xd1, 0x29, 0x36, 0x37, 0xa2, 0xd6, 0x62, 0x07, + 0x1a, 0xd9, 0xb2, 0x46, 0x2f, 0xe9, 0x9c, 0x52, 0x67, 0x91, 0x26, 0xbb, 0x97, 0x5f, 0x7f, 0xb3, + 0xf1, 0xcc, 0xdf, 0xbf, 0xd9, 0x78, 0xe6, 0x57, 0x4f, 0x36, 0x0a, 0x5f, 0x3f, 0xd9, 0x28, 0xfc, + 0xed, 0xc9, 0x46, 0xe1, 0x5f, 0x4f, 0x36, 0x0a, 0x3f, 0xf9, 0xf9, 0xff, 0xf8, 0x1f, 0x21, 0x1a, + 0x07, 0x0c, 0xfb, 0x68, 0xfb, 0x02, 0x53, 0x96, 0x19, 0x0a, 0xcf, 0x87, 0xf2, 0x8f, 0x42, 0x99, + 0xff, 0x0f, 0x71, 0x2d, 0x07, 0xcb, 0x02, 0x7e, 0xeb, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x6c, + 0xf4, 0x1d, 0x49, 0x8c, 0x24, 0x00, 0x00, } func (m *CreateContainerRequest) Marshal() (dAtA []byte, err error) { @@ -5108,6 +5163,43 @@ func (m *SetGuestDateTimeRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) return len(dAtA) - i, nil } +func (m *FSGroup) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FSGroup) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *FSGroup) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.GroupChangePolicy != 0 { + i = encodeVarintAgent(dAtA, i, uint64(m.GroupChangePolicy)) + i-- + dAtA[i] = 0x18 + } + if m.GroupId != 0 { + i = encodeVarintAgent(dAtA, i, uint64(m.GroupId)) + i-- + dAtA[i] = 0x10 + } + return len(dAtA) - i, nil +} + func (m *Storage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -5132,6 +5224,18 @@ func (m *Storage) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.FsGroup != nil { + { + size, err := m.FsGroup.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintAgent(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x3a + } if len(m.MountPoint) > 0 { i -= len(m.MountPoint) copy(dAtA[i:], m.MountPoint) @@ -5452,20 +5556,20 @@ func (m *AddSwapRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { copy(dAtA[i:], m.XXX_unrecognized) } if len(m.PCIPath) > 0 { - dAtA26 := make([]byte, len(m.PCIPath)*10) - var j25 int + dAtA27 := make([]byte, len(m.PCIPath)*10) + var j26 int for _, num := range m.PCIPath { for num >= 1<<7 { - dAtA26[j25] = uint8(uint64(num)&0x7f | 0x80) + dAtA27[j26] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j25++ + j26++ } - dAtA26[j25] = uint8(num) - j25++ + dAtA27[j26] = uint8(num) + j26++ } - i -= j25 - copy(dAtA[i:], dAtA26[:j25]) - i = encodeVarintAgent(dAtA, i, uint64(j25)) + i -= j26 + copy(dAtA[i:], dAtA27[:j26]) + i = encodeVarintAgent(dAtA, i, uint64(j26)) i-- dAtA[i] = 0xa } @@ -6684,6 +6788,24 @@ func (m *SetGuestDateTimeRequest) Size() (n int) { return n } +func (m *FSGroup) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.GroupId != 0 { + n += 1 + sovAgent(uint64(m.GroupId)) + } + if m.GroupChangePolicy != 0 { + n += 1 + sovAgent(uint64(m.GroupChangePolicy)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func (m *Storage) Size() (n int) { if m == nil { return 0 @@ -6718,6 +6840,10 @@ func (m *Storage) Size() (n int) { if l > 0 { n += 1 + l + sovAgent(uint64(l)) } + if m.FsGroup != nil { + l = m.FsGroup.Size() + n += 1 + l + sovAgent(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7631,6 +7757,18 @@ func (this *SetGuestDateTimeRequest) String() string { }, "") return s } +func (this *FSGroup) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&FSGroup{`, + `GroupId:` + fmt.Sprintf("%v", this.GroupId) + `,`, + `GroupChangePolicy:` + fmt.Sprintf("%v", this.GroupChangePolicy) + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} func (this *Storage) String() string { if this == nil { return "nil" @@ -7642,6 +7780,7 @@ func (this *Storage) String() string { `Fstype:` + fmt.Sprintf("%v", this.Fstype) + `,`, `Options:` + fmt.Sprintf("%v", this.Options) + `,`, `MountPoint:` + fmt.Sprintf("%v", this.MountPoint) + `,`, + `FsGroup:` + strings.Replace(this.FsGroup.String(), "FSGroup", "FSGroup", 1) + `,`, `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, `}`, }, "") @@ -14422,6 +14561,95 @@ func (m *SetGuestDateTimeRequest) Unmarshal(dAtA []byte) error { } return nil } +func (m *FSGroup) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FSGroup: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FSGroup: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field GroupId", wireType) + } + m.GroupId = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.GroupId |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field GroupChangePolicy", wireType) + } + m.GroupChangePolicy = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.GroupChangePolicy |= protocols.FSGroupChangePolicy(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipAgent(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Storage) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -14643,6 +14871,42 @@ func (m *Storage) Unmarshal(dAtA []byte) error { } m.MountPoint = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FsGroup", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthAgent + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthAgent + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.FsGroup == nil { + m.FsGroup = &FSGroup{} + } + if err := m.FsGroup.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipAgent(dAtA[iNdEx:]) diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/types.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/types.pb.go index f4dd26cfb2..8646f4bd1b 100644 --- a/src/runtime/virtcontainers/pkg/agent/protocols/types.pb.go +++ b/src/runtime/virtcontainers/pkg/agent/protocols/types.pb.go @@ -49,6 +49,35 @@ func (IPFamily) EnumDescriptor() ([]byte, []int) { return fileDescriptor_f715d0876e8f65d3, []int{0} } +// FSGroupChangePolicy defines the policy for applying group id ownership change on a mounted volume. +type FSGroupChangePolicy int32 + +const ( + // Always indicates that the volume ownership will always be changed. + FSGroupChangePolicy_Always FSGroupChangePolicy = 0 + // OnRootMismatch indicates that the volume ownership will be changed only + // when the ownership of the root directory does not match with the expected group id for the volume. + FSGroupChangePolicy_OnRootMismatch FSGroupChangePolicy = 1 +) + +var FSGroupChangePolicy_name = map[int32]string{ + 0: "Always", + 1: "OnRootMismatch", +} + +var FSGroupChangePolicy_value = map[string]int32{ + "Always": 0, + "OnRootMismatch": 1, +} + +func (x FSGroupChangePolicy) String() string { + return proto.EnumName(FSGroupChangePolicy_name, int32(x)) +} + +func (FSGroupChangePolicy) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_f715d0876e8f65d3, []int{1} +} + type IPAddress struct { Family IPFamily `protobuf:"varint,1,opt,name=family,proto3,enum=types.IPFamily" json:"family,omitempty"` Address string `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"` @@ -230,6 +259,7 @@ var xxx_messageInfo_ARPNeighbor proto.InternalMessageInfo func init() { proto.RegisterEnum("types.IPFamily", IPFamily_name, IPFamily_value) + proto.RegisterEnum("types.FSGroupChangePolicy", FSGroupChangePolicy_name, FSGroupChangePolicy_value) proto.RegisterType((*IPAddress)(nil), "types.IPAddress") proto.RegisterType((*Interface)(nil), "types.Interface") proto.RegisterType((*Route)(nil), "types.Route") @@ -241,38 +271,40 @@ func init() { } var fileDescriptor_f715d0876e8f65d3 = []byte{ - // 482 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x93, 0x3f, 0x8b, 0xdb, 0x30, - 0x18, 0xc6, 0xa3, 0x38, 0xf6, 0xc5, 0x0a, 0xd7, 0x06, 0x51, 0x0e, 0xd1, 0x82, 0x31, 0x59, 0x6a, - 0x0a, 0x8d, 0x21, 0x2d, 0xdd, 0xaf, 0xc3, 0x41, 0x96, 0x62, 0xb4, 0xb5, 0x4b, 0x91, 0x1d, 0xc5, - 0x31, 0xb1, 0x2d, 0x23, 0xc9, 0x09, 0xd9, 0xfa, 0x45, 0xba, 0xf5, 0xc3, 0xdc, 0xd8, 0xb1, 0xe3, - 0x25, 0x9f, 0xa4, 0x48, 0x72, 0x52, 0xf7, 0x0f, 0x85, 0x9b, 0xf2, 0xfe, 0x5e, 0x49, 0x79, 0x9f, - 0xe7, 0x91, 0x05, 0x93, 0xbc, 0x50, 0x9b, 0x36, 0x9d, 0x67, 0xbc, 0x8a, 0xb7, 0x54, 0xd1, 0xd7, - 0x19, 0xaf, 0x15, 0x2d, 0x6a, 0x26, 0xe4, 0x5f, 0x2c, 0x45, 0x16, 0x97, 0x45, 0x2a, 0xe3, 0x46, - 0x70, 0xc5, 0x33, 0x5e, 0x76, 0x95, 0x8c, 0xd5, 0xa1, 0x61, 0x72, 0x6e, 0x00, 0xb9, 0x06, 0x66, - 0x29, 0xf4, 0x97, 0xc9, 0xed, 0x6a, 0x25, 0x98, 0x94, 0xe8, 0x25, 0xf4, 0xd6, 0xb4, 0x2a, 0xca, - 0x03, 0x06, 0x21, 0x88, 0x9e, 0x2c, 0x9e, 0xce, 0xed, 0x89, 0x65, 0x72, 0x67, 0xda, 0xa4, 0x5b, - 0x46, 0x18, 0x5e, 0x51, 0x7b, 0x06, 0x0f, 0x43, 0x10, 0xf9, 0xe4, 0x8c, 0x08, 0xc1, 0x51, 0x45, - 0xe5, 0x16, 0x3b, 0xa6, 0x6d, 0xea, 0xd9, 0x03, 0x80, 0xfe, 0xb2, 0x56, 0x4c, 0xac, 0x69, 0xc6, - 0xd0, 0x0d, 0xf4, 0x56, 0x6c, 0x57, 0x64, 0xcc, 0x0c, 0xf1, 0x49, 0x47, 0xfa, 0x64, 0x4d, 0x2b, - 0xd6, 0xfd, 0xa1, 0xa9, 0xd1, 0x02, 0x4e, 0x2e, 0xea, 0x98, 0xc4, 0x4e, 0xe8, 0x44, 0x93, 0xc5, - 0xf4, 0xa2, 0xaa, 0x5b, 0x21, 0xfd, 0x4d, 0x68, 0x0a, 0x9d, 0x4a, 0xb5, 0x78, 0x14, 0x82, 0x68, - 0x44, 0x74, 0xa9, 0x27, 0x6e, 0xf6, 0x7a, 0x03, 0x76, 0xed, 0x44, 0x4b, 0xda, 0x45, 0x93, 0x15, - 0x09, 0x55, 0x1b, 0xec, 0x59, 0x17, 0x1d, 0x6a, 0x2d, 0x7a, 0x06, 0xbe, 0xb2, 0x5a, 0x74, 0x8d, - 0x5e, 0x40, 0x5f, 0xd0, 0xfd, 0xe7, 0x75, 0x49, 0x73, 0x89, 0xc7, 0x21, 0x88, 0xae, 0xc9, 0x58, - 0xd0, 0xfd, 0x9d, 0xe6, 0xd9, 0x37, 0x00, 0x5d, 0xc2, 0x5b, 0x65, 0x6c, 0xac, 0x98, 0x54, 0x9d, - 0x39, 0x53, 0xeb, 0x41, 0x39, 0x55, 0x6c, 0x4f, 0x0f, 0xe7, 0xb8, 0x3a, 0xec, 0x85, 0xe1, 0xfc, - 0x16, 0xc6, 0x0d, 0xf4, 0x24, 0x6f, 0x45, 0xc6, 0x8c, 0x0f, 0x9f, 0x74, 0x84, 0x9e, 0x41, 0x57, - 0x66, 0xbc, 0x61, 0xc6, 0xc9, 0x35, 0xb1, 0xd0, 0xbb, 0x37, 0xef, 0xbf, 0xf7, 0x36, 0xfb, 0x0a, - 0xe0, 0xe4, 0x96, 0x24, 0x1f, 0x58, 0x91, 0x6f, 0x52, 0x2e, 0x74, 0xbe, 0x8a, 0x5f, 0xc2, 0x33, - 0x9a, 0xff, 0x99, 0x6f, 0x6f, 0x53, 0x4f, 0xf2, 0xf0, 0x4f, 0xc9, 0x65, 0xa9, 0x3f, 0x83, 0xb3, - 0x15, 0x4b, 0x46, 0xb2, 0xa2, 0xca, 0x3a, 0x71, 0x89, 0x05, 0xdd, 0xb5, 0x49, 0xba, 0xb6, 0x6b, - 0xe0, 0xd5, 0x73, 0x38, 0x3e, 0x6b, 0x46, 0x1e, 0x1c, 0xee, 0xde, 0x4e, 0x07, 0xe6, 0xf7, 0xdd, - 0x14, 0xbc, 0x97, 0xf7, 0xc7, 0x60, 0xf0, 0xe3, 0x18, 0x0c, 0xbe, 0x9c, 0x02, 0x70, 0x7f, 0x0a, - 0xc0, 0xf7, 0x53, 0x00, 0x1e, 0x4e, 0x01, 0xf8, 0xf4, 0xf1, 0x91, 0x8f, 0x43, 0xb4, 0xb5, 0x2a, - 0x2a, 0x16, 0xef, 0x0a, 0xa1, 0x7a, 0x4b, 0xcd, 0x36, 0x8f, 0x69, 0xce, 0x6a, 0xf5, 0xeb, 0xe1, - 0xa4, 0x9e, 0x29, 0xdf, 0xfc, 0x0c, 0x00, 0x00, 0xff, 0xff, 0xae, 0x92, 0x74, 0xed, 0x80, 0x03, - 0x00, 0x00, + // 527 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x93, 0xcd, 0x8e, 0xd3, 0x30, + 0x14, 0x85, 0xeb, 0x69, 0x93, 0x69, 0x5c, 0xcd, 0x10, 0x19, 0x34, 0x8a, 0x40, 0x8a, 0xaa, 0x6e, + 0xa8, 0x46, 0xa2, 0x91, 0xca, 0xcf, 0xbe, 0x20, 0x15, 0x75, 0x01, 0x44, 0x66, 0x05, 0x1b, 0xe4, + 0xa6, 0x6e, 0x62, 0x35, 0x89, 0x23, 0xdb, 0x69, 0xd5, 0x1d, 0x2f, 0xc2, 0x8e, 0x87, 0x99, 0x25, + 0x4b, 0x96, 0x33, 0x7d, 0x12, 0x64, 0x3b, 0x2d, 0xe1, 0x47, 0x48, 0xac, 0x7a, 0xbf, 0x6b, 0xbb, + 0xf7, 0x9c, 0xe3, 0x18, 0xc6, 0x29, 0x53, 0x59, 0xbd, 0x9c, 0x24, 0xbc, 0x88, 0x36, 0x44, 0x91, + 0x27, 0x09, 0x2f, 0x15, 0x61, 0x25, 0x15, 0xf2, 0x0f, 0x96, 0x22, 0x89, 0x72, 0xb6, 0x94, 0x51, + 0x25, 0xb8, 0xe2, 0x09, 0xcf, 0x9b, 0x4a, 0x46, 0x6a, 0x5f, 0x51, 0x39, 0x31, 0x80, 0x1c, 0x03, + 0xa3, 0x25, 0xf4, 0x16, 0xf1, 0x6c, 0xb5, 0x12, 0x54, 0x4a, 0xf4, 0x18, 0xba, 0x6b, 0x52, 0xb0, + 0x7c, 0x1f, 0x80, 0x21, 0x18, 0x5f, 0x4e, 0xef, 0x4d, 0xec, 0x89, 0x45, 0x3c, 0x37, 0x6d, 0xdc, + 0x2c, 0xa3, 0x00, 0x9e, 0x13, 0x7b, 0x26, 0x38, 0x1b, 0x82, 0xb1, 0x87, 0x8f, 0x88, 0x10, 0xec, + 0x15, 0x44, 0x6e, 0x82, 0xae, 0x69, 0x9b, 0x7a, 0x74, 0x0b, 0xa0, 0xb7, 0x28, 0x15, 0x15, 0x6b, + 0x92, 0x50, 0x74, 0x05, 0xdd, 0x15, 0xdd, 0xb2, 0x84, 0x9a, 0x21, 0x1e, 0x6e, 0x48, 0x9f, 0x2c, + 0x49, 0x41, 0x9b, 0x3f, 0x34, 0x35, 0x9a, 0xc2, 0xc1, 0x49, 0x1d, 0x95, 0x41, 0x77, 0xd8, 0x1d, + 0x0f, 0xa6, 0xfe, 0x49, 0x55, 0xb3, 0x82, 0xdb, 0x9b, 0x90, 0x0f, 0xbb, 0x85, 0xaa, 0x83, 0xde, + 0x10, 0x8c, 0x7b, 0x58, 0x97, 0x7a, 0x62, 0xb6, 0xd3, 0x1b, 0x02, 0xc7, 0x4e, 0xb4, 0xa4, 0x5d, + 0x54, 0x09, 0x8b, 0x89, 0xca, 0x02, 0xd7, 0xba, 0x68, 0x50, 0x6b, 0xd1, 0x33, 0x82, 0x73, 0xab, + 0x45, 0xd7, 0xe8, 0x11, 0xf4, 0x04, 0xd9, 0x7d, 0x5a, 0xe7, 0x24, 0x95, 0x41, 0x7f, 0x08, 0xc6, + 0x17, 0xb8, 0x2f, 0xc8, 0x6e, 0xae, 0x79, 0xf4, 0x15, 0x40, 0x07, 0xf3, 0x5a, 0x19, 0x1b, 0x2b, + 0x2a, 0x55, 0x63, 0xce, 0xd4, 0x7a, 0x50, 0x4a, 0x14, 0xdd, 0x91, 0xfd, 0x31, 0xae, 0x06, 0x5b, + 0x61, 0x74, 0x7f, 0x09, 0xe3, 0x0a, 0xba, 0x92, 0xd7, 0x22, 0xa1, 0xc6, 0x87, 0x87, 0x1b, 0x42, + 0x0f, 0xa0, 0x23, 0x13, 0x5e, 0x51, 0xe3, 0xe4, 0x02, 0x5b, 0x68, 0xdd, 0x9b, 0xfb, 0xcf, 0x7b, + 0x1b, 0x7d, 0x01, 0x70, 0x30, 0xc3, 0xf1, 0x5b, 0xca, 0xd2, 0x6c, 0xc9, 0x85, 0xce, 0x57, 0xf1, + 0x53, 0x78, 0x46, 0xf3, 0x5f, 0xf3, 0x6d, 0x6d, 0x6a, 0x49, 0x3e, 0xfb, 0x5d, 0x72, 0x9e, 0xeb, + 0xcf, 0xe0, 0x68, 0xc5, 0x92, 0x91, 0xac, 0x88, 0xb2, 0x4e, 0x1c, 0x6c, 0x41, 0x77, 0x6d, 0x92, + 0x8e, 0xed, 0x1a, 0xb8, 0x7e, 0x08, 0xfb, 0x47, 0xcd, 0xc8, 0x85, 0x67, 0xdb, 0x67, 0x7e, 0xc7, + 0xfc, 0xbe, 0xf0, 0xc1, 0xf5, 0x73, 0x78, 0x7f, 0xfe, 0xfe, 0xb5, 0xe0, 0x75, 0xf5, 0x2a, 0x23, + 0x65, 0x4a, 0x63, 0x9e, 0xb3, 0x64, 0x8f, 0x20, 0x74, 0x67, 0xf9, 0x8e, 0xec, 0xa5, 0xdf, 0x41, + 0x08, 0x5e, 0xbe, 0x2b, 0x31, 0xe7, 0xea, 0x0d, 0x93, 0x05, 0x51, 0x49, 0xe6, 0x83, 0x97, 0xf2, + 0xe6, 0x2e, 0xec, 0x7c, 0xbf, 0x0b, 0x3b, 0x9f, 0x0f, 0x21, 0xb8, 0x39, 0x84, 0xe0, 0xdb, 0x21, + 0x04, 0xb7, 0x87, 0x10, 0x7c, 0xfc, 0xf0, 0x9f, 0x6f, 0x4a, 0xd4, 0xa5, 0x62, 0x05, 0x8d, 0xb6, + 0x4c, 0xa8, 0xd6, 0x52, 0xb5, 0x49, 0x23, 0x92, 0xd2, 0x52, 0xfd, 0x7c, 0x6f, 0x4b, 0xd7, 0x94, + 0x4f, 0x7f, 0x04, 0x00, 0x00, 0xff, 0xff, 0xd7, 0x28, 0x03, 0xf1, 0xb7, 0x03, 0x00, 0x00, } func (m *IPAddress) Marshal() (dAtA []byte, err error) { From eff74fab0e4c4a2414166bf18f8de57d1e3e94c7 Mon Sep 17 00:00:00 2001 From: Yibo Zhuang Date: Wed, 6 Apr 2022 19:33:05 -0700 Subject: [PATCH 15/29] agent: fsGroup support for direct-assigned volume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adding two functions set_ownership and recursive_ownership_change to support changing group id ownership for a mounted volume. The set_ownership will be called in common_storage_handler after mount_storage performs the mount for the volume. set_ownership will be a noop if the FSGroup field in the Storage struct is not set which indicates no chown will be performed. If FSGroup field is specified, then it will perform the recursive walk of the mounted volume path to change ownership of all files and directories to the desired group id. It will also configure the SetGid bit so that files created the directory will have group following parent directory group. If the fsGroupChangePolicy is on root mismatch, then the group ownership will be skipped if the root directory group id alreasy matches the desired group id and if the SetGid bit is also set on the root directory. This is the same behavior as what Kubelet does today when performing the recursive walk to change ownership. Fixes #4018 Signed-off-by: Yibo Zhuang (cherry picked from commit 92c00c7e8429a6e6d056b26951a2e052e606cd15) Signed-off-by: Fabiano Fidêncio --- src/agent/src/mount.rs | 311 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 307 insertions(+), 4 deletions(-) diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index f4b50af6ec..b8032fa18e 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -16,7 +16,7 @@ use std::sync::Arc; use tokio::sync::Mutex; use nix::mount::MsFlags; -use nix::unistd::Gid; +use nix::unistd::{Gid, Uid}; use regex::Regex; @@ -29,6 +29,7 @@ use crate::device::{ use crate::linux_abi::*; use crate::pci; use crate::protocols::agent::Storage; +use crate::protocols::types::FSGroupChangePolicy; use crate::Sandbox; #[cfg(target_arch = "s390x")] use crate::{ccw, device::get_virtio_blk_ccw_device_name}; @@ -43,6 +44,11 @@ pub const MOUNT_GUEST_TAG: &str = "kataShared"; // Allocating an FSGroup that owns the pod's volumes const FS_GID: &str = "fsgid"; +const RW_MASK: u32 = 0o660; +const RO_MASK: u32 = 0o440; +const EXEC_MASK: u32 = 0o110; +const MODE_SETGID: u32 = 0o2000; + #[rustfmt::skip] lazy_static! { pub static ref FLAGS: HashMap<&'static str, (bool, MsFlags)> = { @@ -222,7 +228,7 @@ async fn ephemeral_storage_handler( let meta = fs::metadata(&storage.mount_point)?; let mut permission = meta.permissions(); - let o_mode = meta.mode() | 0o2000; + let o_mode = meta.mode() | MODE_SETGID; permission.set_mode(o_mode); fs::set_permissions(&storage.mount_point, permission)?; } @@ -272,7 +278,7 @@ async fn local_storage_handler( if need_set_fsgid { // set SetGid mode mask. - o_mode |= 0o2000; + o_mode |= MODE_SETGID; } permission.set_mode(o_mode); @@ -489,7 +495,9 @@ fn common_storage_handler(logger: &Logger, storage: &Storage) -> Result // Mount the storage device. let mount_point = storage.mount_point.to_string(); - mount_storage(logger, storage).and(Ok(mount_point)) + mount_storage(logger, storage)?; + set_ownership(logger, storage)?; + Ok(mount_point) } // nvdimm_storage_handler handles the storage for NVDIMM driver. @@ -573,6 +581,91 @@ fn mount_storage(logger: &Logger, storage: &Storage) -> Result<()> { ) } +#[instrument] +pub fn set_ownership(logger: &Logger, storage: &Storage) -> Result<()> { + let logger = logger.new(o!("subsystem" => "mount", "fn" => "set_ownership")); + + // If fsGroup is not set, skip performing ownership change + if storage.fs_group.is_none() { + return Ok(()); + } + let fs_group = storage.get_fs_group(); + + let mut read_only = false; + let opts_vec: Vec = storage.options.to_vec(); + if opts_vec.contains(&String::from("ro")) { + read_only = true; + } + + let mount_path = Path::new(&storage.mount_point); + let metadata = mount_path.metadata().map_err(|err| { + error!(logger, "failed to obtain metadata for mount path"; + "mount-path" => mount_path.to_str(), + "error" => err.to_string(), + ); + err + })?; + + if fs_group.group_change_policy == FSGroupChangePolicy::OnRootMismatch + && metadata.gid() == fs_group.group_id + { + let mut mask = if read_only { RO_MASK } else { RW_MASK }; + mask |= EXEC_MASK; + + // With fsGroup change policy to OnRootMismatch, if the current + // gid of the mount path root directory matches the desired gid + // and the current permission of mount path root directory is correct, + // then ownership change will be skipped. + let current_mode = metadata.permissions().mode(); + if (mask & current_mode == mask) && (current_mode & MODE_SETGID != 0) { + info!(logger, "skipping ownership change for volume"; + "mount-path" => mount_path.to_str(), + "fs-group" => fs_group.group_id.to_string(), + ); + return Ok(()); + } + } + + info!(logger, "performing recursive ownership change"; + "mount-path" => mount_path.to_str(), + "fs-group" => fs_group.group_id.to_string(), + ); + recursive_ownership_change( + mount_path, + None, + Some(Gid::from_raw(fs_group.group_id)), + read_only, + ) +} + +#[instrument] +pub fn recursive_ownership_change( + path: &Path, + uid: Option, + gid: Option, + read_only: bool, +) -> Result<()> { + let mut mask = if read_only { RO_MASK } else { RW_MASK }; + if path.is_dir() { + for entry in fs::read_dir(&path)? { + recursive_ownership_change(entry?.path().as_path(), uid, gid, read_only)?; + } + mask |= EXEC_MASK; + mask |= MODE_SETGID; + } + nix::unistd::chown(path, uid, gid)?; + + if gid.is_some() { + let metadata = path.metadata()?; + let mut permission = metadata.permissions(); + let target_mode = metadata.mode() | mask; + permission.set_mode(target_mode); + fs::set_permissions(path, permission)?; + } + + Ok(()) +} + /// Looks for `mount_point` entry in the /proc/mounts. #[instrument] pub fn is_mounted(mount_point: &str) -> Result { @@ -925,6 +1018,8 @@ fn parse_options(option_list: Vec) -> HashMap { mod tests { use super::*; use crate::{skip_if_not_root, skip_loop_if_not_root, skip_loop_if_root}; + use protobuf::RepeatedField; + use protocols::agent::FSGroup; use std::fs::File; use std::fs::OpenOptions; use std::io::Write; @@ -1552,4 +1647,212 @@ mod tests { } } } + + #[test] + fn test_set_ownership() { + skip_if_not_root!(); + + let logger = slog::Logger::root(slog::Discard, o!()); + + #[derive(Debug)] + struct TestData<'a> { + mount_path: &'a str, + fs_group: Option, + read_only: bool, + expected_group_id: u32, + expected_permission: u32, + } + + let tests = &[ + TestData { + mount_path: "foo", + fs_group: None, + read_only: false, + expected_group_id: 0, + expected_permission: 0, + }, + TestData { + mount_path: "rw_mount", + fs_group: Some(FSGroup { + group_id: 3000, + group_change_policy: FSGroupChangePolicy::Always, + unknown_fields: Default::default(), + cached_size: Default::default(), + }), + read_only: false, + expected_group_id: 3000, + expected_permission: RW_MASK | EXEC_MASK | MODE_SETGID, + }, + TestData { + mount_path: "ro_mount", + fs_group: Some(FSGroup { + group_id: 3000, + group_change_policy: FSGroupChangePolicy::OnRootMismatch, + unknown_fields: Default::default(), + cached_size: Default::default(), + }), + read_only: true, + expected_group_id: 3000, + expected_permission: RO_MASK | EXEC_MASK | MODE_SETGID, + }, + ]; + + let tempdir = tempdir().expect("failed to create tmpdir"); + + for (i, d) in tests.iter().enumerate() { + let msg = format!("test[{}]: {:?}", i, d); + + let mount_dir = tempdir.path().join(d.mount_path); + fs::create_dir(&mount_dir) + .unwrap_or_else(|_| panic!("{}: failed to create root directory", msg)); + + let directory_mode = mount_dir.as_path().metadata().unwrap().permissions().mode(); + let mut storage_data = Storage::new(); + if d.read_only { + storage_data.set_options(RepeatedField::from_slice(&[ + "foo".to_string(), + "ro".to_string(), + ])); + } + if let Some(fs_group) = d.fs_group.clone() { + storage_data.set_fs_group(fs_group); + } + storage_data.mount_point = mount_dir.clone().into_os_string().into_string().unwrap(); + + let result = set_ownership(&logger, &storage_data); + assert!(result.is_ok()); + + assert_eq!( + mount_dir.as_path().metadata().unwrap().gid(), + d.expected_group_id + ); + assert_eq!( + mount_dir.as_path().metadata().unwrap().permissions().mode(), + (directory_mode | d.expected_permission) + ); + } + } + + #[test] + fn test_recursive_ownership_change() { + skip_if_not_root!(); + + const COUNT: usize = 5; + + #[derive(Debug)] + struct TestData<'a> { + // Directory where the recursive ownership change should be performed on + path: &'a str, + + // User ID for ownership change + uid: u32, + + // Group ID for ownership change + gid: u32, + + // Set when the permission should be read-only + read_only: bool, + + // The expected permission of all directories after ownership change + expected_permission_directory: u32, + + // The expected permission of all files after ownership change + expected_permission_file: u32, + } + + let tests = &[ + TestData { + path: "no_gid_change", + uid: 0, + gid: 0, + read_only: false, + expected_permission_directory: 0, + expected_permission_file: 0, + }, + TestData { + path: "rw_gid_change", + uid: 0, + gid: 3000, + read_only: false, + expected_permission_directory: RW_MASK | EXEC_MASK | MODE_SETGID, + expected_permission_file: RW_MASK, + }, + TestData { + path: "ro_gid_change", + uid: 0, + gid: 3000, + read_only: true, + expected_permission_directory: RO_MASK | EXEC_MASK | MODE_SETGID, + expected_permission_file: RO_MASK, + }, + ]; + + let tempdir = tempdir().expect("failed to create tmpdir"); + + for (i, d) in tests.iter().enumerate() { + let msg = format!("test[{}]: {:?}", i, d); + + let mount_dir = tempdir.path().join(d.path); + fs::create_dir(&mount_dir) + .unwrap_or_else(|_| panic!("{}: failed to create root directory", msg)); + + let directory_mode = mount_dir.as_path().metadata().unwrap().permissions().mode(); + let mut file_mode: u32 = 0; + + // create testing directories and files + for n in 1..COUNT { + let nest_dir = mount_dir.join(format!("nested{}", n)); + fs::create_dir(&nest_dir) + .unwrap_or_else(|_| panic!("{}: failed to create nest directory", msg)); + + for f in 1..COUNT { + let filename = nest_dir.join(format!("file{}", f)); + File::create(&filename) + .unwrap_or_else(|_| panic!("{}: failed to create file", msg)); + file_mode = filename.as_path().metadata().unwrap().permissions().mode(); + } + } + + let uid = if d.uid > 0 { + Some(Uid::from_raw(d.uid)) + } else { + None + }; + let gid = if d.gid > 0 { + Some(Gid::from_raw(d.gid)) + } else { + None + }; + let result = recursive_ownership_change(&mount_dir, uid, gid, d.read_only); + + assert!(result.is_ok()); + + assert_eq!(mount_dir.as_path().metadata().unwrap().gid(), d.gid); + assert_eq!( + mount_dir.as_path().metadata().unwrap().permissions().mode(), + (directory_mode | d.expected_permission_directory) + ); + + for n in 1..COUNT { + let nest_dir = mount_dir.join(format!("nested{}", n)); + for f in 1..COUNT { + let filename = nest_dir.join(format!("file{}", f)); + let file = Path::new(&filename); + + assert_eq!(file.metadata().unwrap().gid(), d.gid); + assert_eq!( + file.metadata().unwrap().permissions().mode(), + (file_mode | d.expected_permission_file) + ); + } + + let dir = Path::new(&nest_dir); + assert_eq!(dir.metadata().unwrap().gid(), d.gid); + assert_eq!( + dir.metadata().unwrap().permissions().mode(), + (directory_mode | d.expected_permission_directory) + ); + } + } + } } From 78edf827dfde77b607bda54e0ede15414c93a6b3 Mon Sep 17 00:00:00 2001 From: bin Date: Fri, 8 Apr 2022 16:32:29 +0800 Subject: [PATCH 16/29] kata-monitor: add some links when generating pages for browsers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add some links to rendered webpages for better user experience, let users can jump to pages only by clicking links in browsers. Fixes: #4061 Signed-off-by: bin (cherry picked from commit f8cc5d1ad8e1b03754242539c93cb22b4fce0a0d) Signed-off-by: Fabiano Fidêncio --- src/runtime/cmd/kata-monitor/main.go | 35 +++++++++++++++++++++++-- src/runtime/pkg/kata-monitor/metrics.go | 15 +++++++++++ src/runtime/pkg/kata-monitor/monitor.go | 35 +++++++++++++++++++++++++ src/runtime/pkg/kata-monitor/pprof.go | 2 ++ 4 files changed, 85 insertions(+), 2 deletions(-) diff --git a/src/runtime/cmd/kata-monitor/main.go b/src/runtime/cmd/kata-monitor/main.go index 356316dcf2..86b5693d9b 100644 --- a/src/runtime/cmd/kata-monitor/main.go +++ b/src/runtime/cmd/kata-monitor/main.go @@ -175,6 +175,15 @@ func main() { } func indexPage(w http.ResponseWriter, r *http.Request) { + htmlResponse := kataMonitor.IfReturnHTMLResponse(w, r) + if htmlResponse { + indexPageHTML(w, r) + } else { + indexPageText(w, r) + } +} + +func indexPageText(w http.ResponseWriter, r *http.Request) { w.Write([]byte("Available HTTP endpoints:\n")) spacing := 0 @@ -184,13 +193,35 @@ func indexPage(w http.ResponseWriter, r *http.Request) { } } spacing = spacing + 3 + formatter := fmt.Sprintf("%%-%ds: %%s\n", spacing) - formattedString := fmt.Sprintf("%%-%ds: %%s\n", spacing) for _, endpoint := range endpoints { - w.Write([]byte(fmt.Sprintf(formattedString, endpoint.path, endpoint.desc))) + w.Write([]byte(fmt.Sprintf(formatter, endpoint.path, endpoint.desc))) } } +func indexPageHTML(w http.ResponseWriter, r *http.Request) { + + w.Write([]byte("

Available HTTP endpoints:

\n")) + + var formattedString string + needLinkPaths := []string{"/metrics", "/sandboxes"} + + w.Write([]byte("
    ")) + for _, endpoint := range endpoints { + formattedString = fmt.Sprintf("%s: %s\n", endpoint.path, endpoint.desc) + for _, linkPath := range needLinkPaths { + if linkPath == endpoint.path { + formattedString = fmt.Sprintf("%s: %s\n", endpoint.path, endpoint.path, endpoint.desc) + break + } + } + formattedString = fmt.Sprintf("
  • %s
    • ", formattedString) + w.Write([]byte(formattedString)) + } + w.Write([]byte("
")) +} + // initLog setup logger func initLog() { kataMonitorLog := logrus.WithFields(logrus.Fields{ diff --git a/src/runtime/pkg/kata-monitor/metrics.go b/src/runtime/pkg/kata-monitor/metrics.go index 7249906cea..e5d9477670 100644 --- a/src/runtime/pkg/kata-monitor/metrics.go +++ b/src/runtime/pkg/kata-monitor/metrics.go @@ -78,6 +78,21 @@ func (km *KataMonitor) ProcessMetricsRequest(w http.ResponseWriter, r *http.Requ scrapeDurationsHistogram.Observe(float64(time.Since(start).Nanoseconds() / int64(time.Millisecond))) }() + // this is likely the same as `kata-runtime metrics `. + sandboxID, err := getSandboxIDFromReq(r) + if err == nil && sandboxID != "" { + metrics, err := GetSandboxMetrics(sandboxID) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + w.Write([]byte(metrics)) + return + } + + // if no sandbox provided, will get all sandbox's metrics. + // prepare writer for writing response. contentType := expfmt.Negotiate(r.Header) diff --git a/src/runtime/pkg/kata-monitor/monitor.go b/src/runtime/pkg/kata-monitor/monitor.go index ed3ea5c089..1e60a50b7b 100644 --- a/src/runtime/pkg/kata-monitor/monitor.go +++ b/src/runtime/pkg/kata-monitor/monitor.go @@ -27,6 +27,7 @@ const ( RuntimeCRIO = "cri-o" fsMonitorRetryDelaySeconds = 60 podCacheRefreshDelaySeconds = 5 + contentTypeHtml = "text/html" ) // SetLogger sets the logger for katamonitor package. @@ -194,7 +195,41 @@ func (km *KataMonitor) GetAgentURL(w http.ResponseWriter, r *http.Request) { // ListSandboxes list all sandboxes running in Kata func (km *KataMonitor) ListSandboxes(w http.ResponseWriter, r *http.Request) { sandboxes := km.sandboxCache.getSandboxList() + htmlResponse := IfReturnHTMLResponse(w, r) + if htmlResponse { + listSandboxesHtml(sandboxes, w) + } else { + listSandboxesText(sandboxes, w) + } +} + +func listSandboxesText(sandboxes []string, w http.ResponseWriter) { for _, s := range sandboxes { w.Write([]byte(fmt.Sprintf("%s\n", s))) } } +func listSandboxesHtml(sandboxes []string, w http.ResponseWriter) { + w.Write([]byte("

Sandbox list

\n")) + w.Write([]byte("
    \n")) + for _, s := range sandboxes { + w.Write([]byte(fmt.Sprintf("
  • %s: pprof, metrics, agent-url
    • \n", s, s, s, s))) + } + w.Write([]byte("
\n")) +} + +// IfReturnHTMLResponse returns true if request accepts html response +// NOTE: IfReturnHTMLResponse will also set response header to `text/html` +func IfReturnHTMLResponse(w http.ResponseWriter, r *http.Request) bool { + accepts := r.Header["Accept"] + for _, accept := range accepts { + fields := strings.Split(accept, ",") + for _, field := range fields { + if field == contentTypeHtml { + w.Header().Set("Content-Type", contentTypeHtml) + return true + } + } + } + + return false +} diff --git a/src/runtime/pkg/kata-monitor/pprof.go b/src/runtime/pkg/kata-monitor/pprof.go index 62ed70c2e7..0d768e428d 100644 --- a/src/runtime/pkg/kata-monitor/pprof.go +++ b/src/runtime/pkg/kata-monitor/pprof.go @@ -63,8 +63,10 @@ func (km *KataMonitor) proxyRequest(w http.ResponseWriter, r *http.Request, } uri := fmt.Sprintf("http://shim%s", r.URL.String()) + monitorLog.Debugf("proxyRequest to: %s, uri: %s", socketAddress, uri) resp, err := client.Get(uri) if err != nil { + serveError(w, http.StatusInternalServerError, fmt.Sprintf("failed to request %s through %s", uri, socketAddress)) return } From dc1288de8db1a92949cc8cf0f3c36d411b21839a Mon Sep 17 00:00:00 2001 From: Francesco Giudici Date: Wed, 13 Apr 2022 17:46:48 +0200 Subject: [PATCH 17/29] kata-monitor: add a README file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: #3704 Signed-off-by: Francesco Giudici (cherry picked from commit 7b2ff0264710dbb3cb1106cd1568408ebb720532) Signed-off-by: Fabiano Fidêncio --- docs/design/kata-2-0-metrics.md | 1 + src/runtime/README.md | 1 + src/runtime/cmd/kata-monitor/README.md | 68 ++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 src/runtime/cmd/kata-monitor/README.md diff --git a/docs/design/kata-2-0-metrics.md b/docs/design/kata-2-0-metrics.md index 2116ee90a5..e8923a4549 100644 --- a/docs/design/kata-2-0-metrics.md +++ b/docs/design/kata-2-0-metrics.md @@ -51,6 +51,7 @@ The `kata-monitor` management agent should be started on each node where the Kat > **Note**: a *node* running Kata containers will be either a single host system or a worker node belonging to a K8s cluster capable of running Kata pods. - Aggregate sandbox metrics running on the node, adding the `sandbox_id` label to them. +- Attach the additional `cri_uid`, `cri_name` and `cri_namespace` labels to the sandbox metrics, tracking the `uid`, `name` and `namespace` Kubernetes pod metadata. - Expose a new Prometheus target, allowing all node metrics coming from the Kata shim to be collected by Prometheus indirectly. This simplifies the targets count in Prometheus and avoids exposing shim's metrics by `ip:port`. Only one `kata-monitor` process runs in each node. diff --git a/src/runtime/README.md b/src/runtime/README.md index fcffc238ad..50a54c475b 100644 --- a/src/runtime/README.md +++ b/src/runtime/README.md @@ -10,6 +10,7 @@ This repository contains the following components: |-|-| | `containerd-shim-kata-v2` | The [shimv2 runtime](../../docs/design/architecture/README.md#runtime) | | `kata-runtime` | [utility program](../../docs/design/architecture/README.md#utility-program) | +| `kata-monitor` | [metrics collector daemon](cmd/kata-monitor/README.md) | For details of the other Kata Containers repositories, see the [repository summary](https://github.com/kata-containers/kata-containers). diff --git a/src/runtime/cmd/kata-monitor/README.md b/src/runtime/cmd/kata-monitor/README.md new file mode 100644 index 0000000000..81ae2ee0a2 --- /dev/null +++ b/src/runtime/cmd/kata-monitor/README.md @@ -0,0 +1,68 @@ +# Kata monitor + +## Overview +`kata-monitor` is a daemon able to collect and expose metrics related to all the Kata Containers workloads running on the same host. +Once started, it detects all the running Kata Containers runtimes (`containerd-shim-kata-v2`) in the system and exposes few http endpoints to allow the retrieval of the available data. +The main endpoint is the `/metrics` one which aggregates metrics from all the kata workloads. +Available metrics include: + * Kata runtime metrics + * Kata agent metrics + * Kata guest OS metrics + * Hypervisor metrics + * Firecracker metrics + * Kata monitor metrics + +All the provided metrics are in Prometheus format. While `kata-monitor` can be used as a standalone daemon on any host running Kata Containers workloads and can be used for retrieving profiling data from the running Kata runtimes, its main expected usage is to be deployed as a DaemonSet on a Kubernetes cluster: there Prometheus should scrape the metrics from the kata-monitor endpoints. +For more information on the Kata Containers metrics architecture and a detailed list of the available metrics provided by Kata monitor check the [Kata 2.0 Metrics Design](../../../../docs/design/kata-2-0-metrics.md) document. + +## Usage +Each `kata-monitor` instance detects and monitors the Kata Container workloads running on the same node. + +### Kata monitor arguments +The `kata-monitor` binary accepts the following arguments: + +* `--listen-address` _IP:PORT_ +* `--runtime-enpoint` _PATH_TO_THE_CONTAINER_MANAGER_CRI_INTERFACE_ +* `--log-level` _[ trace | debug | info | warn | error | fatal | panic ]_ + +The **listen-address** specifies the IP and TCP port where the kata-monitor HTTP endpoints will be exposed. It defaults to `127.0.0.1:8090`. + +The **runtime-endpoint** is the CRI of a CRI compliant container manager: it will be used to retrieve the CRI `PodSandboxMetadata` (`uid`, `name` and `namespace`) which will be attached to the Kata metrics through the labels `cri_uid`, `cri_name` and `cri_namespace`. It defaults to the containerd socket: `/run/containerd/containerd.sock`. + +The **log-level** allows the chose how verbose the logs should be. The default is `info`. +### Kata monitor HTTP endpoints +`kata-monitor` exposes the following endpoints: + * `/metrics` : get Kata sandboxes metrics. + * `/sandboxes` : list all the Kata sandboxes running on the host. + * `/agent-url` : Get the agent URL of a Kata sandbox. + * `/debug/vars` : Internal data of the Kata runtime shim. + * `/debug/pprof/` : Golang profiling data of the Kata runtime shim: index page. + * `/debug/pprof/cmdline` : Golang profiling data of the Kata runtime shim: `cmdline` endpoint. + * `/debug/pprof/profile` : Golang profiling data of the Kata runtime shim: `profile` endpoint (CPU profiling). + * `/debug/pprof/symbol` : Golang profiling data of the Kata runtime shim: `symbol` endpoint. + * `/debug/pprof/trace` : Golang profiling data of the Kata runtime shim: `trace` endpoint. + +**NOTE: The debug endpoints are available only if the [Kata Containers configuration file](https://github.com/kata-containers/kata-containers/blob/9d5b03a1b70bbd175237ec4b9f821d6ccee0a1f6/src/runtime/config/configuration-qemu.toml.in#L590-L592) includes** `enable_pprof = true` **in the** `[runtime]` **section**. + +The `/sandboxes` endpoint lists the _sandbox ID_ of all the detected Kata runtimes. If accessed via a web browser, it provides html links to the endpoints available for each sandbox. + +In order to retrieve data for a specific Kata workload, the _sandbox ID_ should be passed in the query string using the _sandbox_ key. The `/agent-url`, and all the `/debug/`* endpoints require `sandbox_id` to be specified in the query string. +
+#### Examples +Retrieve the IDs of the available sandboxes: +```bash +$ curl 127.0.0.1:8090/sandboxes +``` +output: +``` +6fcf0a90b01e90d8747177aa466c3462d02e02a878bc393649df83d4c314af0c +df96b24bd49ec437c872c1a758edc084121d607ce1242ff5d2263a0e1b693343 +``` +Retrieve the `agent-url` of the sandbox with ID _df96b24bd49ec437c872c1a758edc084121d607ce1242ff5d2263a0e1b693343_: +```bash +$ curl 127.0.0.1:8090/agent-url?sandbox=df96b24bd49ec437c872c1a758edc084121d607ce1242ff5d2263a0e1b693343 +``` +output: +``` +vsock://830455376:1024 +``` From 688b9abd35b11059f53b6af4f570369e896a1197 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Tue, 5 Apr 2022 14:39:06 +1000 Subject: [PATCH 18/29] runtime: Don't use fixed /tmp/mountPoint path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several tests in kata_agent_test.go create /tmp/mountPoint as a dummy directory to mount. This is not cleaned up after the test. Although it is in /tmp, that's still a little messy and can be confusing to a user. In addition, because it uses the same name every time, it allows for one run of the test to interfere with the next. Use the built in t.TempDir() to use an automatically named and deleted temporary directory instead. Signed-off-by: David Gibson (cherry picked from commit 2eeb5dc2235d43d0cc1a7b8c4c8411621d398cce) Signed-off-by: Fabiano Fidêncio --- src/runtime/virtcontainers/kata_agent_test.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index 3e3b5b6c65..73d3c0765e 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -191,8 +191,7 @@ func TestKataAgentSendReq(t *testing.T) { func TestHandleEphemeralStorage(t *testing.T) { k := kataAgent{} var ociMounts []specs.Mount - mountSource := "/tmp/mountPoint" - os.Mkdir(mountSource, 0755) + mountSource := t.TempDir() mount := specs.Mount{ Type: KataEphemeralDevType, @@ -212,8 +211,7 @@ func TestHandleEphemeralStorage(t *testing.T) { func TestHandleLocalStorage(t *testing.T) { k := kataAgent{} var ociMounts []specs.Mount - mountSource := "/tmp/mountPoint" - os.Mkdir(mountSource, 0755) + mountSource := t.TempDir() mount := specs.Mount{ Type: KataLocalDevType, @@ -688,8 +686,7 @@ func TestHandleShm(t *testing.T) { // In case the type of mount is ephemeral, the container mount is not // shared with the sandbox shm. ociMounts[0].Type = KataEphemeralDevType - mountSource := "/tmp/mountPoint" - os.Mkdir(mountSource, 0755) + mountSource := t.TempDir() ociMounts[0].Source = mountSource k.handleShm(ociMounts, sandbox) From 415420f689ae948da9ba300d3ad96e3c463cccee Mon Sep 17 00:00:00 2001 From: David Gibson Date: Tue, 5 Apr 2022 19:10:10 +1000 Subject: [PATCH 19/29] runtime: Make SetupOCIConfigFile clean up after itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SetupOCIConfigFile creates a temporary directory with os.MkDirTemp(). This means the callers need to register a deferred function to remove it again. At least one of them was commented out meaning that a /temp/katatest- directory was leftover after the unit tests ran. Change to using t.TempDir() which as well as better matching other parts of the tests means the testing framework will handle cleaning it up. Signed-off-by: David Gibson (cherry picked from commit 90b2f5b7760d80134e4c93869e53669f5e870717) Signed-off-by: Fabiano Fidêncio --- src/runtime/pkg/containerd-shim-v2/create_test.go | 6 ------ src/runtime/pkg/containerd-shim-v2/delete_test.go | 5 ++--- src/runtime/pkg/containerd-shim-v2/service_test.go | 3 +-- src/runtime/pkg/katatestutils/utils.go | 5 ++--- src/runtime/pkg/katautils/create_test.go | 12 +++--------- 5 files changed, 8 insertions(+), 23 deletions(-) diff --git a/src/runtime/pkg/containerd-shim-v2/create_test.go b/src/runtime/pkg/containerd-shim-v2/create_test.go index 6b00991f94..6b95aae563 100644 --- a/src/runtime/pkg/containerd-shim-v2/create_test.go +++ b/src/runtime/pkg/containerd-shim-v2/create_test.go @@ -50,7 +50,6 @@ func TestCreateSandboxSuccess(t *testing.T) { }() tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t) - // defer os.RemoveAll(tmpdir) runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true) assert.NoError(err) @@ -99,7 +98,6 @@ func TestCreateSandboxFail(t *testing.T) { assert := assert.New(t) tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t) - defer os.RemoveAll(tmpdir) runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true) assert.NoError(err) @@ -137,7 +135,6 @@ func TestCreateSandboxConfigFail(t *testing.T) { assert := assert.New(t) tmpdir, bundlePath, _ := ktu.SetupOCIConfigFile(t) - defer os.RemoveAll(tmpdir) runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true) assert.NoError(err) @@ -187,7 +184,6 @@ func TestCreateContainerSuccess(t *testing.T) { } tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t) - defer os.RemoveAll(tmpdir) runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true) assert.NoError(err) @@ -227,7 +223,6 @@ func TestCreateContainerFail(t *testing.T) { assert := assert.New(t) tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t) - defer os.RemoveAll(tmpdir) runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true) assert.NoError(err) @@ -278,7 +273,6 @@ func TestCreateContainerConfigFail(t *testing.T) { }() tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t) - defer os.RemoveAll(tmpdir) runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true) assert.NoError(err) diff --git a/src/runtime/pkg/containerd-shim-v2/delete_test.go b/src/runtime/pkg/containerd-shim-v2/delete_test.go index f84f5e596e..7d959e15a5 100644 --- a/src/runtime/pkg/containerd-shim-v2/delete_test.go +++ b/src/runtime/pkg/containerd-shim-v2/delete_test.go @@ -7,7 +7,6 @@ package containerdshim import ( - "os" "testing" taskAPI "github.com/containerd/containerd/runtime/v2/task" @@ -25,8 +24,8 @@ func TestDeleteContainerSuccessAndFail(t *testing.T) { MockID: testSandboxID, } - rootPath, bundlePath, _ := ktu.SetupOCIConfigFile(t) - defer os.RemoveAll(rootPath) + _, bundlePath, _ := ktu.SetupOCIConfigFile(t) + _, err := compatoci.ParseConfigJSON(bundlePath) assert.NoError(err) diff --git a/src/runtime/pkg/containerd-shim-v2/service_test.go b/src/runtime/pkg/containerd-shim-v2/service_test.go index b501df99cc..f895b4e852 100644 --- a/src/runtime/pkg/containerd-shim-v2/service_test.go +++ b/src/runtime/pkg/containerd-shim-v2/service_test.go @@ -41,8 +41,7 @@ func TestServiceCreate(t *testing.T) { assert := assert.New(t) - tmpdir, bundleDir, _ := ktu.SetupOCIConfigFile(t) - defer os.RemoveAll(tmpdir) + _, bundleDir, _ := ktu.SetupOCIConfigFile(t) ctx := context.Background() diff --git a/src/runtime/pkg/katatestutils/utils.go b/src/runtime/pkg/katatestutils/utils.go index 7750c23deb..527c9cfbc7 100644 --- a/src/runtime/pkg/katatestutils/utils.go +++ b/src/runtime/pkg/katatestutils/utils.go @@ -346,11 +346,10 @@ func IsInGitHubActions() bool { func SetupOCIConfigFile(t *testing.T) (rootPath string, bundlePath, ociConfigFile string) { assert := assert.New(t) - tmpdir, err := os.MkdirTemp("", "katatest-") - assert.NoError(err) + tmpdir := t.TempDir() bundlePath = filepath.Join(tmpdir, "bundle") - err = os.MkdirAll(bundlePath, testDirMode) + err := os.MkdirAll(bundlePath, testDirMode) assert.NoError(err) ociConfigFile = filepath.Join(bundlePath, "config.json") diff --git a/src/runtime/pkg/katautils/create_test.go b/src/runtime/pkg/katautils/create_test.go index e2488aaa93..b9fc795530 100644 --- a/src/runtime/pkg/katautils/create_test.go +++ b/src/runtime/pkg/katautils/create_test.go @@ -216,7 +216,6 @@ func TestCreateSandboxConfigFail(t *testing.T) { assert := assert.New(t) tmpdir, bundlePath, _ := ktu.SetupOCIConfigFile(t) - defer os.RemoveAll(tmpdir) runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true) assert.NoError(err) @@ -250,7 +249,6 @@ func TestCreateSandboxFail(t *testing.T) { assert := assert.New(t) tmpdir, bundlePath, _ := ktu.SetupOCIConfigFile(t) - defer os.RemoveAll(tmpdir) runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true) assert.NoError(err) @@ -273,7 +271,6 @@ func TestCreateSandboxAnnotations(t *testing.T) { assert := assert.New(t) tmpdir, bundlePath, _ := ktu.SetupOCIConfigFile(t) - defer os.RemoveAll(tmpdir) runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true) assert.NoError(err) @@ -350,8 +347,7 @@ func TestCheckForFips(t *testing.T) { func TestCreateContainerContainerConfigFail(t *testing.T) { assert := assert.New(t) - tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t) - defer os.RemoveAll(tmpdir) + _, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t) spec, err := compatoci.ParseConfigJSON(bundlePath) assert.NoError(err) @@ -378,8 +374,7 @@ func TestCreateContainerContainerConfigFail(t *testing.T) { func TestCreateContainerFail(t *testing.T) { assert := assert.New(t) - tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t) - defer os.RemoveAll(tmpdir) + _, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t) spec, err := compatoci.ParseConfigJSON(bundlePath) assert.NoError(err) @@ -413,8 +408,7 @@ func TestCreateContainer(t *testing.T) { mockSandbox.CreateContainerFunc = nil }() - tmpdir, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t) - defer os.RemoveAll(tmpdir) + _, bundlePath, ociConfigFile := ktu.SetupOCIConfigFile(t) spec, err := compatoci.ParseConfigJSON(bundlePath) assert.NoError(err) From 4e350f7d53484dc466ab7c08485cb5f34718763d Mon Sep 17 00:00:00 2001 From: David Gibson Date: Tue, 5 Apr 2022 21:01:28 +1000 Subject: [PATCH 20/29] runtime: Clean up mock hook logs in tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tests in hook_test.go run a mock hook binary, which does some debug logging to /tmp/mock_hook.log. Currently we don't clean up those logs when the tests are done. Use a test cleanup function to do this. Signed-off-by: David Gibson (cherry picked from commit f7ba21c86fc78fc36699204f4333df49995d5ebf) Signed-off-by: Fabiano Fidêncio --- src/runtime/pkg/katautils/hook_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/runtime/pkg/katautils/hook_test.go b/src/runtime/pkg/katautils/hook_test.go index 50452974b6..f8900cc4dd 100644 --- a/src/runtime/pkg/katautils/hook_test.go +++ b/src/runtime/pkg/katautils/hook_test.go @@ -22,6 +22,7 @@ var testContainerIDHook = "test-container-id" var testControllerIDHook = "test-controller-id" var testBinHookPath = "../../virtcontainers/hook/mock/hook" var testBundlePath = "/test/bundle" +var mockHookLogFile = "/tmp/mock_hook.log" func getMockHookBinPath() string { return testBinHookPath @@ -49,12 +50,17 @@ func createWrongHook() specs.Hook { } } +func cleanMockHookLogFile() { + _ = os.Remove(mockHookLogFile) +} + func TestRunHook(t *testing.T) { if tc.NotValid(ktu.NeedRoot()) { t.Skip(ktu.TestDisabledNeedRoot) } assert := assert.New(t) + t.Cleanup(cleanMockHookLogFile) ctx := context.Background() spec := specs.Spec{} @@ -87,6 +93,7 @@ func TestPreStartHooks(t *testing.T) { } assert := assert.New(t) + t.Cleanup(cleanMockHookLogFile) ctx := context.Background() @@ -129,6 +136,7 @@ func TestPostStartHooks(t *testing.T) { } assert := assert.New(t) + t.Cleanup(cleanMockHookLogFile) ctx := context.Background() @@ -173,6 +181,7 @@ func TestPostStopHooks(t *testing.T) { assert := assert.New(t) ctx := context.Background() + t.Cleanup(cleanMockHookLogFile) // Hooks field is nil spec := specs.Spec{} From c49084f303c2e5122190f725841238c43c27be4a Mon Sep 17 00:00:00 2001 From: David Gibson Date: Wed, 6 Apr 2022 12:10:59 +1000 Subject: [PATCH 21/29] runtime: Make bind mount tests better clean up after themselves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are several tests in mount_test.go which perform a sample bind mount. These need a corresponding unmount to clean up afterwards or attempting to delete the temporary files will fail due to the existing mountpoint. Most of them had such an unmount, but TestBindMountInvalidPgtypes was missing one. In addition, the existing unmounts where done inconsistently - one was simply inline (so wouldn't be executed if the test fails too early) and one is a defer. Change them all to use the t.Cleanup mechanism. For the dummy mountpoint files, rather than cleaning them up after the test, the tests were removing them at the beginning of the test. That stops the test being messed up by a previous run, but messily. Since these are created in a private temporary directory anyway, if there's something already there, that indicates a problem we shouldn't ignore. In fact we don't need to explicitly remove these at all - they'll be removed along with the rest of the private temporary directory. Signed-off-by: David Gibson (cherry picked from commit bec59f9e398247d3946443c6c57f731f185a9b03) Signed-off-by: Fabiano Fidêncio --- src/runtime/virtcontainers/mount_test.go | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/runtime/virtcontainers/mount_test.go b/src/runtime/virtcontainers/mount_test.go index 6564a9648f..00bd290010 100644 --- a/src/runtime/virtcontainers/mount_test.go +++ b/src/runtime/virtcontainers/mount_test.go @@ -383,6 +383,12 @@ func TestBindMountFailingMount(t *testing.T) { assert.Error(err) } +func cleanupFooMount() { + dest := filepath.Join(testDir, "fooDirDest") + + syscall.Unmount(dest, 0) +} + func TestBindMountSuccessful(t *testing.T) { assert := assert.New(t) if tc.NotValid(ktu.NeedRoot()) { @@ -391,9 +397,7 @@ func TestBindMountSuccessful(t *testing.T) { source := filepath.Join(testDir, "fooDirSrc") dest := filepath.Join(testDir, "fooDirDest") - syscall.Unmount(dest, 0) - os.Remove(source) - os.Remove(dest) + t.Cleanup(cleanupFooMount) err := os.MkdirAll(source, mountPerm) assert.NoError(err) @@ -403,8 +407,6 @@ func TestBindMountSuccessful(t *testing.T) { err = bindMount(context.Background(), source, dest, false, "private") assert.NoError(err) - - syscall.Unmount(dest, 0) } func TestBindMountReadonlySuccessful(t *testing.T) { @@ -415,9 +417,7 @@ func TestBindMountReadonlySuccessful(t *testing.T) { source := filepath.Join(testDir, "fooDirSrc") dest := filepath.Join(testDir, "fooDirDest") - syscall.Unmount(dest, 0) - os.Remove(source) - os.Remove(dest) + t.Cleanup(cleanupFooMount) err := os.MkdirAll(source, mountPerm) assert.NoError(err) @@ -428,8 +428,6 @@ func TestBindMountReadonlySuccessful(t *testing.T) { err = bindMount(context.Background(), source, dest, true, "private") assert.NoError(err) - defer syscall.Unmount(dest, 0) - // should not be able to create file in read-only mount destFile := filepath.Join(dest, "foo") _, err = os.OpenFile(destFile, os.O_CREATE, mountPerm) @@ -444,9 +442,7 @@ func TestBindMountInvalidPgtypes(t *testing.T) { source := filepath.Join(testDir, "fooDirSrc") dest := filepath.Join(testDir, "fooDirDest") - syscall.Unmount(dest, 0) - os.Remove(source) - os.Remove(dest) + t.Cleanup(cleanupFooMount) err := os.MkdirAll(source, mountPerm) assert.NoError(err) From e1c4f57c353dff6ff15b62331b10f0b1f661151b Mon Sep 17 00:00:00 2001 From: David Gibson Date: Wed, 6 Apr 2022 15:29:06 +1000 Subject: [PATCH 22/29] runtime: Don't abuse MockStorageRootPath() for factory tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A number of unit tests under virtcontainers/factory use MockStorageRootPath() as a general purpose temporary directory. This doesn't make sense: the mockfs driver isn't even in use here since we only call EnableMockTesting for the pase virtcontainers package, not the subpackages. Instead use t.TempDir() which is for exactly this purpose. As a bonus it also handles the cleanup, so we don't need MockStorageDestroy any more. Signed-off-by: David Gibson (cherry picked from commit 1719a8b49152d4ee5f3e944324fd9dd1f70bbbb9) Signed-off-by: Fabiano Fidêncio --- .../virtcontainers/factory/cache/cache_test.go | 4 +--- .../factory/direct/direct_test.go | 4 +--- .../virtcontainers/factory/factory_test.go | 18 +++++++----------- .../factory/template/template_test.go | 4 +--- .../virtcontainers/persist/fs/mockfs.go | 4 ---- 5 files changed, 10 insertions(+), 24 deletions(-) diff --git a/src/runtime/virtcontainers/factory/cache/cache_test.go b/src/runtime/virtcontainers/factory/cache/cache_test.go index d0971ef12b..9f2fa3a5b5 100644 --- a/src/runtime/virtcontainers/factory/cache/cache_test.go +++ b/src/runtime/virtcontainers/factory/cache/cache_test.go @@ -13,14 +13,12 @@ import ( vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/factory/direct" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/fs" ) func TestTemplateFactory(t *testing.T) { assert := assert.New(t) - testDir := fs.MockStorageRootPath() - defer fs.MockStorageDestroy() + testDir := t.TempDir() hyperConfig := vc.HypervisorConfig{ KernelPath: testDir, diff --git a/src/runtime/virtcontainers/factory/direct/direct_test.go b/src/runtime/virtcontainers/factory/direct/direct_test.go index 724b6cb66f..862fcb6365 100644 --- a/src/runtime/virtcontainers/factory/direct/direct_test.go +++ b/src/runtime/virtcontainers/factory/direct/direct_test.go @@ -12,14 +12,12 @@ import ( "github.com/stretchr/testify/assert" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/fs" ) func TestTemplateFactory(t *testing.T) { assert := assert.New(t) - testDir := fs.MockStorageRootPath() - defer fs.MockStorageDestroy() + testDir := t.TempDir() hyperConfig := vc.HypervisorConfig{ KernelPath: testDir, diff --git a/src/runtime/virtcontainers/factory/factory_test.go b/src/runtime/virtcontainers/factory/factory_test.go index 71af52d71e..80cff101df 100644 --- a/src/runtime/virtcontainers/factory/factory_test.go +++ b/src/runtime/virtcontainers/factory/factory_test.go @@ -12,7 +12,6 @@ import ( vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/factory/base" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/fs" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/mock" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" "github.com/sirupsen/logrus" @@ -39,10 +38,10 @@ func TestNewFactory(t *testing.T) { _, err = NewFactory(ctx, config, false) assert.Error(err) - defer fs.MockStorageDestroy() + testDir := t.TempDir() config.VMConfig.HypervisorConfig = vc.HypervisorConfig{ - KernelPath: fs.MockStorageRootPath(), - ImagePath: fs.MockStorageRootPath(), + KernelPath: testDir, + ImagePath: testDir, } // direct @@ -69,7 +68,7 @@ func TestNewFactory(t *testing.T) { defer hybridVSockTTRPCMock.Stop() config.Template = true - config.TemplatePath = fs.MockStorageRootPath() + config.TemplatePath = testDir f, err = NewFactory(ctx, config, false) assert.Nil(err) f.CloseFactory(ctx) @@ -134,8 +133,7 @@ func TestCheckVMConfig(t *testing.T) { err = checkVMConfig(config1, config2) assert.Nil(err) - testDir := fs.MockStorageRootPath() - defer fs.MockStorageDestroy() + testDir := t.TempDir() config1.HypervisorConfig = vc.HypervisorConfig{ KernelPath: testDir, @@ -155,8 +153,7 @@ func TestCheckVMConfig(t *testing.T) { func TestFactoryGetVM(t *testing.T) { assert := assert.New(t) - testDir := fs.MockStorageRootPath() - defer fs.MockStorageDestroy() + testDir := t.TempDir() hyperConfig := vc.HypervisorConfig{ KernelPath: testDir, @@ -321,8 +318,7 @@ func TestDeepCompare(t *testing.T) { config.VMConfig = vc.VMConfig{ HypervisorType: vc.MockHypervisor, } - testDir := fs.MockStorageRootPath() - defer fs.MockStorageDestroy() + testDir := t.TempDir() config.VMConfig.HypervisorConfig = vc.HypervisorConfig{ KernelPath: testDir, diff --git a/src/runtime/virtcontainers/factory/template/template_test.go b/src/runtime/virtcontainers/factory/template/template_test.go index 55c7bc5968..c067c793e6 100644 --- a/src/runtime/virtcontainers/factory/template/template_test.go +++ b/src/runtime/virtcontainers/factory/template/template_test.go @@ -16,7 +16,6 @@ import ( "github.com/stretchr/testify/assert" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/fs" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/mock" ) @@ -32,8 +31,7 @@ func TestTemplateFactory(t *testing.T) { templateWaitForAgent = 1 * time.Microsecond - testDir := fs.MockStorageRootPath() - defer fs.MockStorageDestroy() + testDir := t.TempDir() hyperConfig := vc.HypervisorConfig{ KernelPath: testDir, diff --git a/src/runtime/virtcontainers/persist/fs/mockfs.go b/src/runtime/virtcontainers/persist/fs/mockfs.go index f972782cf4..23e09c8c04 100644 --- a/src/runtime/virtcontainers/persist/fs/mockfs.go +++ b/src/runtime/virtcontainers/persist/fs/mockfs.go @@ -30,10 +30,6 @@ func MockRunVMStoragePath() string { return filepath.Join(MockStorageRootPath(), vmPathSuffix) } -func MockStorageDestroy() { - os.RemoveAll(MockStorageRootPath()) -} - func MockFSInit() (persistapi.PersistDriver, error) { driver, err := Init() if err != nil { From aed4fe6a2eb82c5bac71a5191d4aef36b11e821e Mon Sep 17 00:00:00 2001 From: David Gibson Date: Wed, 6 Apr 2022 13:11:56 +1000 Subject: [PATCH 23/29] runtime: Export StoragePathSuffix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit storagePathSuffix defines the file path suffix - "vc" - used for Kata's persistent storage information, as a private constant. We duplicate this information in fc.go which also needs it. Export it from fs.go instead, so it can be used in fc.go. Signed-off-by: David Gibson (cherry picked from commit 963d03ea8a7cf9d9dc298c5c6c80e98a0dec7606) Signed-off-by: Fabiano Fidêncio --- src/runtime/virtcontainers/fc.go | 5 ++--- src/runtime/virtcontainers/persist/fs/fs.go | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/runtime/virtcontainers/fc.go b/src/runtime/virtcontainers/fc.go index 6137e32cf2..4630a810c8 100644 --- a/src/runtime/virtcontainers/fc.go +++ b/src/runtime/virtcontainers/fc.go @@ -27,6 +27,7 @@ import ( hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/fs" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client" models "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/models" ops "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/operations" @@ -84,8 +85,6 @@ const ( fcMetricsFifo = "metrics.fifo" defaultFcConfig = "fcConfig.json" - // storagePathSuffix mirrors persist/fs/fs.go:storagePathSuffix - storagePathSuffix = "vc" ) // Specify the minimum version of firecracker supported @@ -244,7 +243,7 @@ func (fc *firecracker) setPaths(hypervisorConfig *HypervisorConfig) { // /// hypervisorName := filepath.Base(hypervisorConfig.HypervisorPath) //fs.RunStoragePath cannot be used as we need exec perms - fc.chrootBaseDir = filepath.Join("/run", storagePathSuffix) + fc.chrootBaseDir = filepath.Join("/run", fs.StoragePathSuffix) fc.vmPath = filepath.Join(fc.chrootBaseDir, hypervisorName, fc.id) fc.jailerRoot = filepath.Join(fc.vmPath, "root") // auto created by jailer diff --git a/src/runtime/virtcontainers/persist/fs/fs.go b/src/runtime/virtcontainers/persist/fs/fs.go index 630ed76364..407f765568 100644 --- a/src/runtime/virtcontainers/persist/fs/fs.go +++ b/src/runtime/virtcontainers/persist/fs/fs.go @@ -29,11 +29,11 @@ const dirMode = os.FileMode(0700) | os.ModeDir // fileMode is the permission bits used for creating a file const fileMode = os.FileMode(0600) -// storagePathSuffix is the suffix used for all storage paths +// StoragePathSuffix is the suffix used for all storage paths // // Note: this very brief path represents "virtcontainers". It is as // terse as possible to minimise path length. -const storagePathSuffix = "vc" +const StoragePathSuffix = "vc" // sandboxPathSuffix is the suffix used for sandbox storage const sandboxPathSuffix = "sbs" @@ -64,7 +64,7 @@ func Init() (persistapi.PersistDriver, error) { return &FS{ sandboxState: &persistapi.SandboxState{}, containerState: make(map[string]persistapi.ContainerState), - storageRootPath: filepath.Join("/run", storagePathSuffix), + storageRootPath: filepath.Join("/run", StoragePathSuffix), driverName: "fs", }, nil } From 3e74243fbed789e189272d1a065716478180a0b9 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Wed, 6 Apr 2022 14:46:44 +1000 Subject: [PATCH 24/29] runtime: Move mockfs control global into mockfs.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit virtcontainers/persist/fs/mockfs.go defines a mock filesystem type for testing. A global variable in virtcontainers/persist/manager.go is used to force use of the mock fs rather than a normal one. This patch moves the global, and the EnableMockTesting() function which sets it into mockfs.go. This is slightly cleaner to begin with, and will allow some further enhancements. Signed-off-by: David Gibson (cherry picked from commit 5d8438e9397c5f44ad460bbb695501c46b3616da) Signed-off-by: Fabiano Fidêncio --- .../virtcontainers/persist/fs/mockfs.go | 13 +++++++ .../virtcontainers/persist/fs/mockfs_test.go | 34 +++++++++++++++++++ src/runtime/virtcontainers/persist/manager.go | 10 ++---- .../virtcontainers/persist/manager_test.go | 14 -------- .../virtcontainers/virtcontainers_test.go | 3 +- 5 files changed, 51 insertions(+), 23 deletions(-) create mode 100644 src/runtime/virtcontainers/persist/fs/mockfs_test.go diff --git a/src/runtime/virtcontainers/persist/fs/mockfs.go b/src/runtime/virtcontainers/persist/fs/mockfs.go index 23e09c8c04..4c4f492693 100644 --- a/src/runtime/virtcontainers/persist/fs/mockfs.go +++ b/src/runtime/virtcontainers/persist/fs/mockfs.go @@ -13,11 +13,17 @@ import ( persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" ) +var mockTesting = false + type MockFS struct { // inherit from FS. Overwrite if needed. *FS } +func EnableMockTesting() { + mockTesting = true +} + func MockStorageRootPath() string { return filepath.Join(os.TempDir(), "vc", "mockfs") } @@ -46,3 +52,10 @@ func MockFSInit() (persistapi.PersistDriver, error) { return &MockFS{fsDriver}, nil } + +func MockAutoInit() (persistapi.PersistDriver, error) { + if mockTesting { + return MockFSInit() + } + return nil, nil +} diff --git a/src/runtime/virtcontainers/persist/fs/mockfs_test.go b/src/runtime/virtcontainers/persist/fs/mockfs_test.go new file mode 100644 index 0000000000..99709a78e6 --- /dev/null +++ b/src/runtime/virtcontainers/persist/fs/mockfs_test.go @@ -0,0 +1,34 @@ +// Copyright Red Hat. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package fs + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMockAutoInit(t *testing.T) { + assert := assert.New(t) + orgMockTesting := mockTesting + defer func() { + mockTesting = orgMockTesting + }() + + mockTesting = false + + fsd, err := MockAutoInit() + assert.Nil(fsd) + assert.NoError(err) + + // Testing mock driver + mockTesting = true + fsd, err = MockAutoInit() + assert.NoError(err) + expectedFS, err := MockFSInit() + assert.NoError(err) + assert.Equal(expectedFS, fsd) +} diff --git a/src/runtime/virtcontainers/persist/manager.go b/src/runtime/virtcontainers/persist/manager.go index a104bdcabf..32504c932b 100644 --- a/src/runtime/virtcontainers/persist/manager.go +++ b/src/runtime/virtcontainers/persist/manager.go @@ -28,13 +28,8 @@ var ( RootFSName: fs.Init, RootlessFSName: fs.RootlessInit, } - mockTesting = false ) -func EnableMockTesting() { - mockTesting = true -} - // GetDriver returns new PersistDriver according to driver name func GetDriverByName(name string) (persistapi.PersistDriver, error) { if expErr != nil { @@ -56,8 +51,9 @@ func GetDriver() (persistapi.PersistDriver, error) { return nil, expErr } - if mockTesting { - return fs.MockFSInit() + mock, err := fs.MockAutoInit() + if mock != nil || err != nil { + return mock, err } if rootless.IsRootless() { diff --git a/src/runtime/virtcontainers/persist/manager_test.go b/src/runtime/virtcontainers/persist/manager_test.go index 074ca92665..4347f9adc2 100644 --- a/src/runtime/virtcontainers/persist/manager_test.go +++ b/src/runtime/virtcontainers/persist/manager_test.go @@ -27,12 +27,6 @@ func TestGetDriverByName(t *testing.T) { func TestGetDriver(t *testing.T) { assert := assert.New(t) - orgMockTesting := mockTesting - defer func() { - mockTesting = orgMockTesting - }() - - mockTesting = false fsd, err := GetDriver() assert.NoError(err) @@ -46,12 +40,4 @@ func TestGetDriver(t *testing.T) { assert.NoError(err) assert.Equal(expectedFS, fsd) - - // Testing mock driver - mockTesting = true - fsd, err = GetDriver() - assert.NoError(err) - expectedFS, err = fs.MockFSInit() - assert.NoError(err) - assert.Equal(expectedFS, fsd) } diff --git a/src/runtime/virtcontainers/virtcontainers_test.go b/src/runtime/virtcontainers/virtcontainers_test.go index cb03e2351b..6a3d7fa580 100644 --- a/src/runtime/virtcontainers/virtcontainers_test.go +++ b/src/runtime/virtcontainers/virtcontainers_test.go @@ -15,7 +15,6 @@ import ( "syscall" "testing" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/fs" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" "github.com/sirupsen/logrus" @@ -108,7 +107,7 @@ func setupClh() { func TestMain(m *testing.M) { var err error - persist.EnableMockTesting() + fs.EnableMockTesting() flag.Parse() From 0bd1abac3e5a1dc36fa8fd739b3d994f99bb6d5b Mon Sep 17 00:00:00 2001 From: David Gibson Date: Wed, 6 Apr 2022 15:02:48 +1000 Subject: [PATCH 25/29] runtime: Let MockFSInit create a mock fs driver at any path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently MockFSInit always creates the mockfs at the fixed path /tmp/vc/mockfs. This change allows it to be initialized at any path given as a parameter. This allows the tests in fs_test.go to be simplified, because the by using a temporary directory from t.TempDir(), which is automatically cleaned up, we don't need to manually trigger initTestDir() (which is misnamed, it's actually a cleanup function). For now we still use the fixed path when auto-creating the mockfs in MockAutoInit(), but we'll change that later. Signed-off-by: David Gibson (cherry picked from commit ef6d54a78106f2a1c8e8c25dae3bb91f2d76cf65) Signed-off-by: Fabiano Fidêncio --- .../virtcontainers/persist/fs/fs_test.go | 26 +++++-------------- .../virtcontainers/persist/fs/mockfs.go | 6 ++--- .../virtcontainers/persist/fs/mockfs_test.go | 2 +- 3 files changed, 10 insertions(+), 24 deletions(-) diff --git a/src/runtime/virtcontainers/persist/fs/fs_test.go b/src/runtime/virtcontainers/persist/fs/fs_test.go index 91af4af159..370b87c527 100644 --- a/src/runtime/virtcontainers/persist/fs/fs_test.go +++ b/src/runtime/virtcontainers/persist/fs/fs_test.go @@ -14,8 +14,8 @@ import ( "github.com/stretchr/testify/assert" ) -func getFsDriver() (*FS, error) { - driver, err := MockFSInit() +func getFsDriver(t *testing.T) (*FS, error) { + driver, err := MockFSInit(t.TempDir()) if err != nil { return nil, fmt.Errorf("failed to init fs driver") } @@ -27,16 +27,8 @@ func getFsDriver() (*FS, error) { return fs.FS, nil } -func initTestDir() func() { - return func() { - os.RemoveAll(MockStorageRootPath()) - } -} - func TestFsLockShared(t *testing.T) { - defer initTestDir()() - - fs, err := getFsDriver() + fs, err := getFsDriver(t) assert.Nil(t, err) assert.NotNil(t, fs) @@ -61,9 +53,7 @@ func TestFsLockShared(t *testing.T) { } func TestFsLockExclusive(t *testing.T) { - defer initTestDir()() - - fs, err := getFsDriver() + fs, err := getFsDriver(t) assert.Nil(t, err) assert.NotNil(t, fs) @@ -89,9 +79,7 @@ func TestFsLockExclusive(t *testing.T) { } func TestFsDriver(t *testing.T) { - defer initTestDir()() - - fs, err := getFsDriver() + fs, err := getFsDriver(t) assert.Nil(t, err) assert.NotNil(t, fs) @@ -162,12 +150,10 @@ func TestFsDriver(t *testing.T) { } func TestGlobalReadWrite(t *testing.T) { - defer initTestDir()() - relPath := "test/123/aaa.json" data := "hello this is testing global read write" - fs, err := getFsDriver() + fs, err := getFsDriver(t) assert.Nil(t, err) assert.NotNil(t, fs) diff --git a/src/runtime/virtcontainers/persist/fs/mockfs.go b/src/runtime/virtcontainers/persist/fs/mockfs.go index 4c4f492693..dca4acad42 100644 --- a/src/runtime/virtcontainers/persist/fs/mockfs.go +++ b/src/runtime/virtcontainers/persist/fs/mockfs.go @@ -36,7 +36,7 @@ func MockRunVMStoragePath() string { return filepath.Join(MockStorageRootPath(), vmPathSuffix) } -func MockFSInit() (persistapi.PersistDriver, error) { +func MockFSInit(rootPath string) (persistapi.PersistDriver, error) { driver, err := Init() if err != nil { return nil, fmt.Errorf("Could not create Mock FS driver: %v", err) @@ -47,7 +47,7 @@ func MockFSInit() (persistapi.PersistDriver, error) { return nil, fmt.Errorf("Could not create Mock FS driver") } - fsDriver.storageRootPath = MockStorageRootPath() + fsDriver.storageRootPath = rootPath fsDriver.driverName = "mockfs" return &MockFS{fsDriver}, nil @@ -55,7 +55,7 @@ func MockFSInit() (persistapi.PersistDriver, error) { func MockAutoInit() (persistapi.PersistDriver, error) { if mockTesting { - return MockFSInit() + return MockFSInit(MockStorageRootPath()) } return nil, nil } diff --git a/src/runtime/virtcontainers/persist/fs/mockfs_test.go b/src/runtime/virtcontainers/persist/fs/mockfs_test.go index 99709a78e6..9cd5832895 100644 --- a/src/runtime/virtcontainers/persist/fs/mockfs_test.go +++ b/src/runtime/virtcontainers/persist/fs/mockfs_test.go @@ -28,7 +28,7 @@ func TestMockAutoInit(t *testing.T) { mockTesting = true fsd, err = MockAutoInit() assert.NoError(err) - expectedFS, err := MockFSInit() + expectedFS, err := MockFSInit(MockStorageRootPath()) assert.NoError(err) assert.Equal(expectedFS, fsd) } From 3f5c6e7182b798504df1e2cacf4f7d7d813f5655 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Wed, 6 Apr 2022 15:12:49 +1000 Subject: [PATCH 26/29] runtime: Allock mockfs storage to be placed in any directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently EnableMockTesting() takes no arguments and will always place the mock storage in the fixed location /tmp/vc/mockfs. This means that one test run can interfere with the next one if anything isn't cleaned up (and there are other bugs which means that happens). If if those were fixed this would allow developers testing on the same machine to interfere with each other. So, allow the mockfs to be placed at an arbitrary place given as a parameter to EnableMockTesting(). In TestMain() we place it under our existing temporary directory, so we don't need any additional cleanup just for the mockfs. fixes #4140 Signed-off-by: David Gibson (cherry picked from commit 1b931f420339f8849e98517b38ba609a7fa7c9d3) Signed-off-by: Fabiano Fidêncio --- src/runtime/virtcontainers/persist/fs/mockfs.go | 14 ++++++++------ .../virtcontainers/persist/fs/mockfs_test.go | 8 ++++---- src/runtime/virtcontainers/virtcontainers_test.go | 6 ++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/runtime/virtcontainers/persist/fs/mockfs.go b/src/runtime/virtcontainers/persist/fs/mockfs.go index dca4acad42..18045f1ee5 100644 --- a/src/runtime/virtcontainers/persist/fs/mockfs.go +++ b/src/runtime/virtcontainers/persist/fs/mockfs.go @@ -7,25 +7,27 @@ package fs import ( "fmt" - "os" "path/filepath" persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" ) -var mockTesting = false +var mockRootPath = "" type MockFS struct { // inherit from FS. Overwrite if needed. *FS } -func EnableMockTesting() { - mockTesting = true +func EnableMockTesting(rootPath string) { + mockRootPath = rootPath } func MockStorageRootPath() string { - return filepath.Join(os.TempDir(), "vc", "mockfs") + if mockRootPath == "" { + panic("Using uninitialized mock storage root path") + } + return mockRootPath } func MockRunStoragePath() string { @@ -54,7 +56,7 @@ func MockFSInit(rootPath string) (persistapi.PersistDriver, error) { } func MockAutoInit() (persistapi.PersistDriver, error) { - if mockTesting { + if mockRootPath != "" { return MockFSInit(MockStorageRootPath()) } return nil, nil diff --git a/src/runtime/virtcontainers/persist/fs/mockfs_test.go b/src/runtime/virtcontainers/persist/fs/mockfs_test.go index 9cd5832895..f2c1abd766 100644 --- a/src/runtime/virtcontainers/persist/fs/mockfs_test.go +++ b/src/runtime/virtcontainers/persist/fs/mockfs_test.go @@ -13,19 +13,19 @@ import ( func TestMockAutoInit(t *testing.T) { assert := assert.New(t) - orgMockTesting := mockTesting + orgMockRootPath := mockRootPath defer func() { - mockTesting = orgMockTesting + mockRootPath = orgMockRootPath }() - mockTesting = false + mockRootPath = "" fsd, err := MockAutoInit() assert.Nil(fsd) assert.NoError(err) // Testing mock driver - mockTesting = true + mockRootPath = t.TempDir() fsd, err = MockAutoInit() assert.NoError(err) expectedFS, err := MockFSInit(MockStorageRootPath()) diff --git a/src/runtime/virtcontainers/virtcontainers_test.go b/src/runtime/virtcontainers/virtcontainers_test.go index 6a3d7fa580..b7117000ce 100644 --- a/src/runtime/virtcontainers/virtcontainers_test.go +++ b/src/runtime/virtcontainers/virtcontainers_test.go @@ -57,8 +57,6 @@ var testHyperstartTtySocket = "" // cleanUp Removes any stale sandbox/container state that can affect // the next test to run. func cleanUp() { - os.RemoveAll(fs.MockRunStoragePath()) - os.RemoveAll(fs.MockRunVMStoragePath()) syscall.Unmount(GetSharePath(testSandboxID), syscall.MNT_DETACH|UmountNoFollow) os.RemoveAll(testDir) os.MkdirAll(testDir, DirMode) @@ -107,8 +105,6 @@ func setupClh() { func TestMain(m *testing.M) { var err error - fs.EnableMockTesting() - flag.Parse() logger := logrus.NewEntry(logrus.New()) @@ -125,6 +121,8 @@ func TestMain(m *testing.M) { panic(err) } + fs.EnableMockTesting(filepath.Join(testDir, "mockfs")) + fmt.Printf("INFO: Creating virtcontainers test directory %s\n", testDir) err = os.MkdirAll(testDir, DirMode) if err != nil { From 8b21c5f78d017bd0c8d14f2044c1b63f7190a5a6 Mon Sep 17 00:00:00 2001 From: holyfei Date: Wed, 20 Apr 2022 15:04:54 +0800 Subject: [PATCH 27/29] agent: modify the type of swappiness to u64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The type of MemorySwappiness in runtime is uint64, and the type of swappiness in agent is int64, if we set max uint64 in runtime and pass it to agent, the value will be equal to -1. We should modify the type of swappiness to u64 Fixes: #4123 Signed-off-by: holyfei (cherry picked from commit 02395027816a0ac4e287a1e4d9c7a72dc2a6f8cd) Signed-off-by: Fabiano Fidêncio --- src/agent/rustjail/src/cgroups/fs/mod.rs | 2 +- src/agent/rustjail/src/lib.rs | 2 +- src/libs/oci/src/lib.rs | 2 +- src/tools/agent-ctl/src/utils.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs index 035bb8a6b4..84aa9bc50c 100644 --- a/src/agent/rustjail/src/cgroups/fs/mod.rs +++ b/src/agent/rustjail/src/cgroups/fs/mod.rs @@ -391,7 +391,7 @@ fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool if let Some(swappiness) = memory.swappiness { if (0..=100).contains(&swappiness) { - mem_controller.set_swappiness(swappiness as u64)?; + mem_controller.set_swappiness(swappiness)?; } else { return Err(anyhow!( "invalid value:{}. valid memory swappiness range is 0-100", diff --git a/src/agent/rustjail/src/lib.rs b/src/agent/rustjail/src/lib.rs index 7535bf9901..2e4c7ff54b 100644 --- a/src/agent/rustjail/src/lib.rs +++ b/src/agent/rustjail/src/lib.rs @@ -265,7 +265,7 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources swap: Some(mem.Swap), kernel: Some(mem.Kernel), kernel_tcp: Some(mem.KernelTCP), - swappiness: Some(mem.Swappiness as i64), + swappiness: Some(mem.Swappiness), disable_oom_killer: Some(mem.DisableOOMKiller), }) } else { diff --git a/src/libs/oci/src/lib.rs b/src/libs/oci/src/lib.rs index 69a2a98e15..f47f2df4be 100644 --- a/src/libs/oci/src/lib.rs +++ b/src/libs/oci/src/lib.rs @@ -381,7 +381,7 @@ pub struct LinuxMemory { #[serde(default, skip_serializing_if = "Option::is_none", rename = "kernelTCP")] pub kernel_tcp: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub swappiness: Option, + pub swappiness: Option, #[serde( default, skip_serializing_if = "Option::is_none", diff --git a/src/tools/agent-ctl/src/utils.rs b/src/tools/agent-ctl/src/utils.rs index 8dbfa53f4e..064b54486a 100644 --- a/src/tools/agent-ctl/src/utils.rs +++ b/src/tools/agent-ctl/src/utils.rs @@ -400,7 +400,7 @@ fn memory_oci_to_ttrpc( Swap: mem.swap.unwrap_or(0), Kernel: mem.kernel.unwrap_or(0), KernelTCP: mem.kernel_tcp.unwrap_or(0), - Swappiness: mem.swappiness.unwrap_or(0) as u64, + Swappiness: mem.swappiness.unwrap_or(0), DisableOOMKiller: mem.disable_oom_killer.unwrap_or(false), unknown_fields: protobuf::UnknownFields::new(), cached_size: protobuf::CachedSize::default(), From 81e10fe34fff76b33f0a0fa7c999439ce78722bc Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Thu, 21 Apr 2022 16:47:41 +0200 Subject: [PATCH 28/29] packaging: Fix clh build from source fall-back MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If we fail to download the clh binary, we fall-back to build from source. Unfortunately, `pull_clh_released_binary()` leaves a `cloud_hypervisor` directory behind, which causes `build_clh_from_source()` not to clone the git repo: [ -d "${repo_dir}" ] || git clone "${cloud_hypervisor_repo}" When building from a kata-containers git repo, the subsequent calls to `git` in this function thus apply to the kata-containers repo and eventually fail, e.g.: + git checkout v23.0 error: pathspec 'v23.0' did not match any file(s) known to git It doesn't quite make sense actually to keep an existing directory the content of which is arbitrary when we want to it to contain a specific version of clh. Just remove it instead. Fixes: #4151 Signed-off-by: Greg Kurz (cherry picked from commit afbd60da2787fc739c677633dc1c967d9dd08184) Signed-off-by: Fabiano Fidêncio --- .../static-build/cloud-hypervisor/build-static-clh.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh index cf2aaec381..d88b03c0d6 100755 --- a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh +++ b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh @@ -48,7 +48,8 @@ build_clh_from_source() { info "Build ${cloud_hypervisor_repo} version: ${cloud_hypervisor_version}" repo_dir=$(basename "${cloud_hypervisor_repo}") repo_dir="${repo_dir//.git}" - [ -d "${repo_dir}" ] || git clone "${cloud_hypervisor_repo}" + rm -rf "${repo_dir}" + git clone "${cloud_hypervisor_repo}" pushd "${repo_dir}" git fetch || true git checkout "${cloud_hypervisor_version}" From 6b2c641f0b4207b7f37fd0809c30cdfa42394759 Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Thu, 21 Apr 2022 15:39:16 +0200 Subject: [PATCH 29/29] tools: fix typo in clh directory name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows to get released binaries again. Fixes: #4151 Signed-off-by: Greg Kurz (cherry picked from commit b658dccc5f336da3c264a5944ce968ef6f03df9b) Signed-off-by: Fabiano Fidêncio --- .../packaging/static-build/cloud-hypervisor/build-static-clh.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh index d88b03c0d6..24d3ec84c2 100755 --- a/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh +++ b/tools/packaging/static-build/cloud-hypervisor/build-static-clh.sh @@ -41,7 +41,7 @@ pull_clh_released_binary() { curl --fail -L ${cloud_hypervisor_binary} -o cloud-hypervisor-static || return 1 mkdir -p cloud-hypervisor mv -f cloud-hypervisor-static cloud-hypervisor/cloud-hypervisor - chmod +x cloud_hypervisor/cloud-hypervisor + chmod +x cloud-hypervisor/cloud-hypervisor } build_clh_from_source() {