From 4a54876dde2a1de91977e27df9241941a2962198 Mon Sep 17 00:00:00 2001 From: Ji-Xinyou Date: Mon, 25 Jul 2022 17:40:44 +0800 Subject: [PATCH 01/26] runtime-rs: support static resource management functionality Supports functionalities of static resource management, enabled by default. Fixes: #4742 Signed-off-by: Ji-Xinyou --- .../src/annotations/cri_containerd.rs | 11 ++ src/libs/kata-types/src/annotations/mod.rs | 33 ++++ src/libs/kata-types/src/config/runtime.rs | 4 + src/runtime-rs/Makefile | 4 +- .../config/configuration-dragonball.toml.in | 3 + .../crates/hypervisor/src/dragonball/inner.rs | 1 + src/runtime-rs/crates/runtimes/src/lib.rs | 1 + src/runtime-rs/crates/runtimes/src/manager.rs | 19 ++ .../crates/runtimes/src/static_resource.rs | 167 ++++++++++++++++++ 9 files changed, 241 insertions(+), 2 deletions(-) create mode 100644 src/runtime-rs/crates/runtimes/src/static_resource.rs diff --git a/src/libs/kata-types/src/annotations/cri_containerd.rs b/src/libs/kata-types/src/annotations/cri_containerd.rs index db6462a8c8..8b2d63fafd 100644 --- a/src/libs/kata-types/src/annotations/cri_containerd.rs +++ b/src/libs/kata-types/src/annotations/cri_containerd.rs @@ -11,3 +11,14 @@ pub const SANDBOX: &str = "sandbox"; pub const CONTAINER: &str = "container"; pub const SANDBOX_ID_LABEL_KEY: &str = "io.kubernetes.cri.sandbox-id"; + +// Ref: https://pkg.go.dev/github.com/containerd/containerd@v1.6.7/pkg/cri/annotations +// SandboxCPU annotations are based on the initial CPU configuration for the sandbox. This is calculated as the +// sum of container CPU resources, optionally provided by Kubelet (introduced in 1.23) as part of the PodSandboxConfig +pub const SANDBOX_CPU_QUOTA_KEY: &str = "io.kubernetes.cri.sandbox-cpu-quota"; +pub const SANDBOX_CPU_PERIOD_KEY: &str = "io.kubernetes.cri.sandbox-cpu-period"; +pub const SANDBOX_CPU_SHARE_KEY: &str = "io.kubernetes.cri.sandbox-cpu-shares"; + +// SandboxMemory is the initial amount of memory associated with this sandbox. This is calculated as the sum +// of container memory, optionally provided by Kubelet (introduced in 1.23) as part of the PodSandboxConfig +pub const SANDBOX_MEM_KEY: &str = "io.kubernetes.cri.sandbox-memory"; diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index 0a517e2216..4ae67a9f06 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -16,6 +16,8 @@ use crate::config::hypervisor::get_hypervisor_plugin; use crate::config::TomlConfig; use crate::sl; +use self::cri_containerd::{SANDBOX_CPU_PERIOD_KEY, SANDBOX_CPU_QUOTA_KEY, SANDBOX_MEM_KEY}; + /// CRI-containerd specific annotations. pub mod cri_containerd; @@ -376,6 +378,37 @@ impl Annotation { self.get(CONTAINER_TYPE_KEY) } + /// Get the annotation of cpu quota for sandbox + pub fn get_sandbox_cpu_quota(&self) -> u64 { + let value = self + .get_value::(SANDBOX_CPU_QUOTA_KEY) + .unwrap_or(Some(0)); + if let Some(q) = value { + return q; + } + 0 + } + + /// Get the annotation of cpu period for sandbox + pub fn get_sandbox_cpu_period(&self) -> i64 { + let value = self + .get_value::(SANDBOX_CPU_PERIOD_KEY) + .unwrap_or(Some(0)); + if let Some(p) = value { + return p; + } + 0 + } + + /// Get the annotation of memory for sandbox + pub fn get_sandbox_mem(&self) -> i64 { + let value = self.get_value::(SANDBOX_MEM_KEY).unwrap_or(Some(0)); + if let Some(m) = value { + return m; + } + 0 + } + /// Get the annotation to specify the Resources.Memory.Swappiness. pub fn get_container_resource_swappiness(&self) -> Result> { match self.get_value::(KATA_ANNO_CONTAINER_RES_SWAPPINESS) { diff --git a/src/libs/kata-types/src/config/runtime.rs b/src/libs/kata-types/src/config/runtime.rs index ce8e9efa59..a9fa3de9db 100644 --- a/src/libs/kata-types/src/config/runtime.rs +++ b/src/libs/kata-types/src/config/runtime.rs @@ -99,6 +99,10 @@ pub struct Runtime { #[serde(default)] pub enable_pprof: bool, + /// If enabled, static resource management will calculate the vcpu and memory for the sandbox/container + #[serde(default)] + pub static_resource_mgmt: bool, + /// Determines whether container seccomp profiles are passed to the virtual machine and /// applied by the kata agent. If set to true, seccomp is not applied within the guest. #[serde(default)] diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 32c2214a04..a7ae59ed2c 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -107,7 +107,7 @@ DEFMSIZE9P := 8192 DEFVFIOMODE := guest-kernel # Default cgroup model DEFSANDBOXCGROUPONLY ?= false -DEFSTATICRESOURCEMGMT ?= false +DEFSTATICRESOURCEMGMT_DB ?= true DEFBINDMOUNTS := [] SED = sed CLI_DIR = cmd @@ -240,7 +240,7 @@ USER_VARS += DEFMSIZE9P USER_VARS += DEFENTROPYSOURCE USER_VARS += DEFVALIDENTROPYSOURCES USER_VARS += DEFSANDBOXCGROUPONLY -USER_VARS += DEFSTATICRESOURCEMGMT +USER_VARS += DEFSTATICRESOURCEMGMT_DB USER_VARS += DEFBINDMOUNTS USER_VARS += DEFVFIOMODE USER_VARS += BUILDFLAGS diff --git a/src/runtime-rs/config/configuration-dragonball.toml.in b/src/runtime-rs/config/configuration-dragonball.toml.in index bda6a8d3a1..6efb18ccf6 100644 --- a/src/runtime-rs/config/configuration-dragonball.toml.in +++ b/src/runtime-rs/config/configuration-dragonball.toml.in @@ -247,3 +247,6 @@ experimental=@DEFAULTEXPFEATURES@ # If enabled, user can run pprof tools with shim v2 process through kata-monitor. # (default: false) # enable_pprof = true + +static_resource_mgmt=@DEFSTATICRESOURCEMGMT_DB@ + diff --git a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs index aef8d3352d..8a053830f2 100644 --- a/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs @@ -173,6 +173,7 @@ impl DragonballInner { serial_path: Some(serial_path), mem_size_mib: self.config.memory_info.default_memory as usize, vcpu_count: self.config.cpu_info.default_vcpus as u8, + max_vcpu_count: self.config.cpu_info.default_maxvcpus as u8, ..Default::default() }; info!(sl!(), "vm config: {:?}", vm_config); diff --git a/src/runtime-rs/crates/runtimes/src/lib.rs b/src/runtime-rs/crates/runtimes/src/lib.rs index 64c57feeae..1b8c96478a 100644 --- a/src/runtime-rs/crates/runtimes/src/lib.rs +++ b/src/runtime-rs/crates/runtimes/src/lib.rs @@ -11,3 +11,4 @@ logging::logger_with_subsystem!(sl, "runtimes"); mod manager; pub use manager::RuntimeHandlerManager; +mod static_resource; diff --git a/src/runtime-rs/crates/runtimes/src/manager.rs b/src/runtime-rs/crates/runtimes/src/manager.rs index 10a4a427bb..dba10579a8 100644 --- a/src/runtime-rs/crates/runtimes/src/manager.rs +++ b/src/runtime-rs/crates/runtimes/src/manager.rs @@ -7,6 +7,8 @@ use std::sync::Arc; use anyhow::{anyhow, Context, Result}; + +use crate::static_resource::StaticResourceManager; use common::{ message::Message, types::{Request, Response}, @@ -279,6 +281,23 @@ fn load_config(spec: &oci::Spec) -> Result { let (mut toml_config, _) = TomlConfig::load_from_file(&config_path).context("load toml config")?; annotation.update_config_by_annotation(&mut toml_config)?; + + // Sandbox sizing information *may* be provided in two scenarios: + // 1. The upper layer runtime (ie, containerd or crio) provide sandbox sizing information as an annotation + // in the 'sandbox container's' spec. This would typically be a scenario where as part of a create sandbox + // request the upper layer runtime receives this information as part of a pod, and makes it available to us + // for sizing purposes. + // 2. If this is not a sandbox infrastructure container, but instead a standalone single container (analogous to "docker run..."), + // then the container spec itself will contain appropriate sizing information for the entire sandbox (since it is + // a single container. + if toml_config.runtime.static_resource_mgmt { + info!(sl!(), "static resource management enabled"); + let static_resource_manager = StaticResourceManager::new(spec) + .context("failed to construct static resource manager")?; + static_resource_manager + .setup_config(&mut toml_config) + .context("failed to setup static resource mgmt config")?; + } info!(sl!(), "get config content {:?}", &toml_config); Ok(toml_config) } diff --git a/src/runtime-rs/crates/runtimes/src/static_resource.rs b/src/runtime-rs/crates/runtimes/src/static_resource.rs new file mode 100644 index 0000000000..0e04d21505 --- /dev/null +++ b/src/runtime-rs/crates/runtimes/src/static_resource.rs @@ -0,0 +1,167 @@ +// Copyright (c) 2019-2021 Alibaba Cloud +// Copyright (c) 2019-2021 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +use std::convert::TryFrom; + +use anyhow::{Context, Result}; + +use kata_types::{ + annotations::Annotation, config::TomlConfig, container::ContainerType, + cpu::LinuxContainerCpuResources, k8s::container_type, +}; + +// static resource that StaticResourceManager needs, this is the spec for the +// sandbox/container's workload +#[derive(Clone, Copy, Debug)] +struct StaticResource { + vcpu: u32, + mem_mb: u32, +} + +// generate static resource(vcpu and memory in MiB) from spec's information +// used for static resource management +impl TryFrom<&oci::Spec> for StaticResource { + type Error = anyhow::Error; + fn try_from(spec: &oci::Spec) -> Result { + let mut vcpu: u32 = 0; + let mut mem_mb: u32 = 0; + match container_type(spec) { + // podsandbox, from annotation + ContainerType::PodSandbox => { + let annotation = Annotation::new(spec.annotations.clone()); + let (period, quota, memory) = + get_sizing_info(annotation).context("failed to get sizing info")?; + let cpu = oci::LinuxCpu { + period: Some(period), + quota: Some(quota), + ..Default::default() + }; + // although it may not be actually a linux container, we are only using the calculation inside + // LinuxContainerCpuResources::try_from to generate our vcpu number + if let Ok(cpu_resource) = LinuxContainerCpuResources::try_from(&cpu) { + vcpu = get_nr_vcpu(&cpu_resource); + } + mem_mb = convert_memory_to_mb(memory); + } + // single container, from container spec + _ => { + if let Some(linux) = &spec.linux { + if let Some(resource) = &linux.resources { + if let Some(cpu) = &resource.cpu { + if let Ok(cpu_resource) = LinuxContainerCpuResources::try_from(cpu) { + vcpu = get_nr_vcpu(&cpu_resource); + } + } + if let Some(mem) = &resource.memory { + let memory = mem.limit.unwrap_or(0); + mem_mb = convert_memory_to_mb(memory); + } + } + } + } + } + info!( + sl!(), + "static resource mgmt result: vcpu={}, mem_mb={}", vcpu, mem_mb + ); + Ok(Self { vcpu, mem_mb }) + } +} + +// StaticResourceManager is responsible for static resource management +// +// static resource management sizing information is optionally provided, either by +// upper layer runtime (containerd / crio) or by the container spec itself (when it +// is a standalone single container such as the one started with *docker run*) +// +// the sizing information uses three values, cpu quota, cpu period and memory limit, +// and with above values it calculates the # vcpus and memory for the workload and +// add them to default value of the config +#[derive(Clone, Copy, Debug)] +pub struct StaticResourceManager { + resource: StaticResource, +} + +impl StaticResourceManager { + pub fn new(spec: &oci::Spec) -> Result { + Ok(Self { + resource: StaticResource::try_from(spec) + .context("failed to construct static resource")?, + }) + } + + pub fn setup_config(&self, config: &mut TomlConfig) -> Result<()> { + // update this data to the hypervisor config for later use by hypervisor + let hypervisor_name = &config.runtime.hypervisor_name; + let mut hv = config + .hypervisor + .get_mut(hypervisor_name) + .context("failed to get hypervisor config")?; + hv.cpu_info.default_vcpus += self.resource.vcpu as i32; + hv.memory_info.default_memory += self.resource.mem_mb; + Ok(()) + } +} + +fn get_nr_vcpu(resource: &LinuxContainerCpuResources) -> u32 { + if let Some(v) = resource.get_vcpus() { + v as u32 + } else { + 0 + } +} + +fn convert_memory_to_mb(memory_in_byte: i64) -> u32 { + if memory_in_byte < 0 { + 0 + } else { + (memory_in_byte / 1024 / 1024) as u32 + } +} + +// from the upper layer runtime's annotation (e.g. crio, k8s), get the *cpu quota, +// cpu period and memory limit* for a sandbox/container +fn get_sizing_info(annotation: Annotation) -> Result<(u64, i64, i64)> { + // since we are *adding* our result to the config, a value of 0 will cause no change + // and if the annotation is not assigned (but static resource management is), we will + // log a *warning* to fill that with zero value + let period = annotation.get_sandbox_cpu_quota(); + let quota = annotation.get_sandbox_cpu_period(); + let memory = annotation.get_sandbox_mem(); + Ok((period, quota, memory)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_static_resource_mgmt() { + // data below should result in 2200 mCPU(round up to 3 vcpus) and 512 MiB of memory + let period: u64 = 100000; + let memory: i64 = 1048576 * 512; // 512 MiB + let quota: i64 = 220000; + + let cpu = oci::LinuxCpu { + period: Some(period), + quota: Some(quota), + ..Default::default() + }; + if let Ok(cpu_resource) = LinuxContainerCpuResources::try_from(&cpu) { + if let Some(v) = cpu_resource.get_vcpus() { + assert_eq!(v, 3); + } + } + + let mem_mb = if memory < 0 { + 0 + } else { + (memory / 1024 / 1024) as u32 + }; + + assert_eq!(mem_mb, 512); + } +} From 00f3a6de12b6a2e20694cd0557d93e02d4de4b3f Mon Sep 17 00:00:00 2001 From: Ji-Xinyou Date: Mon, 15 Aug 2022 11:14:18 +0800 Subject: [PATCH 02/26] runtime-rs: make static resource mgmt idiomatic Make the get value process (cpu and mem) more idiomatic. Fixes: #4742 Signed-off-by: Ji-Xinyou --- src/libs/kata-types/src/annotations/mod.rs | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index 4ae67a9f06..07cdbe1a1d 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -383,10 +383,7 @@ impl Annotation { let value = self .get_value::(SANDBOX_CPU_QUOTA_KEY) .unwrap_or(Some(0)); - if let Some(q) = value { - return q; - } - 0 + value.unwrap_or(0) } /// Get the annotation of cpu period for sandbox @@ -394,19 +391,13 @@ impl Annotation { let value = self .get_value::(SANDBOX_CPU_PERIOD_KEY) .unwrap_or(Some(0)); - if let Some(p) = value { - return p; - } - 0 + value.unwrap_or(0) } /// Get the annotation of memory for sandbox pub fn get_sandbox_mem(&self) -> i64 { let value = self.get_value::(SANDBOX_MEM_KEY).unwrap_or(Some(0)); - if let Some(m) = value { - return m; - } - 0 + value.unwrap_or(0) } /// Get the annotation to specify the Resources.Memory.Swappiness. From ff7c78e0e8956801999d17ed10fab11d604756f2 Mon Sep 17 00:00:00 2001 From: Ji-Xinyou Date: Mon, 15 Aug 2022 14:42:38 +0800 Subject: [PATCH 03/26] runtime-rs: static resource mgmt default to false Static resource management should be default to false. If default to be true, later update sandbox operation, e.g. resize, will not work. Fixes: #4742 Signed-off-by: Ji-Xinyou --- src/runtime-rs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index a7ae59ed2c..21f571fb15 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -107,7 +107,7 @@ DEFMSIZE9P := 8192 DEFVFIOMODE := guest-kernel # Default cgroup model DEFSANDBOXCGROUPONLY ?= false -DEFSTATICRESOURCEMGMT_DB ?= true +DEFSTATICRESOURCEMGMT_DB ?= false DEFBINDMOUNTS := [] SED = sed CLI_DIR = cmd From b1a8acad5724b45646f0ee778efb24bd17445012 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Tue, 30 Aug 2022 15:33:56 +0000 Subject: [PATCH 04/26] versions: Update cni plugins version This PR updates the cni plugins version that is being used in the kata CI. Fixes #5039 Depends-on: github.com/kata-containers/tests#5088 Signed-off-by: Gabriela Cervantes --- versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.yaml b/versions.yaml index 164969397b..40ed46589c 100644 --- a/versions.yaml +++ b/versions.yaml @@ -180,7 +180,7 @@ externals: cni-plugins: description: "CNI network plugins" url: "https://github.com/containernetworking/plugins" - commit: "485be65581341430f9106a194a98f0f2412245fb" + version: "v1.1.1" conmon: description: "An OCI container runtime monitor" From bed4aab7eea0f173d0ab5d243bc2facd03464d7e Mon Sep 17 00:00:00 2001 From: Derek Lee Date: Mon, 1 Aug 2022 11:09:19 -0700 Subject: [PATCH 05/26] github-actions: Add cargo-deny Adds cargo-deny to scan for vulnerabilities and license issues regarding rust crates. GitHub Actions does not have an obvious way to loop over each of the Cargo.toml files. To avoid hardcoding it, I worked around the problem using a composite action that first generates the cargo-deny action by finding all Cargo.toml files before calling this new generated action in the master workflow. Uses recommended deny.toml from cargo-deny repo with the following modifications: ignore = ["RUSTSEC-2020-0071"] because chrono is dependent on the version of time with the vulnerability and there is no simple workaround multiple-versions = "allow" Because of the above error and other packages, there are instances where some crates require different versions of a crate. unknown-git = "allow" I don't see a particular issue with allowing crates from other repos. An alternative would be the manually set each repo we want in an allow-git list, but I see this as more of a nuisance that its worth. We could leave this as a warning (default), but to avoid clutter I'm going to allow it. If deny.toml needs to be edited in the future, here's the guide: https://embarkstudios.github.io/cargo-deny/index.html Fixes #3359 Signed-off-by: Derek Lee --- .../cargo-deny-generator.sh | 40 +++++++++++++++++++ .../cargo-deny-skeleton.yaml.in | 30 ++++++++++++++ .github/workflows/cargo-deny-runner.yaml | 19 +++++++++ ci/lib.sh | 10 +++++ deny.toml | 33 +++++++++++++++ 5 files changed, 132 insertions(+) create mode 100644 .github/cargo-deny-composite-action/cargo-deny-generator.sh create mode 100644 .github/cargo-deny-composite-action/cargo-deny-skeleton.yaml.in create mode 100644 .github/workflows/cargo-deny-runner.yaml create mode 100644 deny.toml diff --git a/.github/cargo-deny-composite-action/cargo-deny-generator.sh b/.github/cargo-deny-composite-action/cargo-deny-generator.sh new file mode 100644 index 0000000000..3d9eba242c --- /dev/null +++ b/.github/cargo-deny-composite-action/cargo-deny-generator.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# +# Copyright (c) 2022 Red Hat +# +# SPDX-License-Identifier: Apache-2.0 +# + +script_dir=$(dirname "$(readlink -f "$0")") +parent_dir=$(realpath "${script_dir}/../..") +cidir="${parent_dir}/ci" +source "${cidir}/lib.sh" + +cargo_deny_file="${script_dir}/action.yaml" + +cat cargo-deny-skeleton.yaml.in > "${cargo_deny_file}" + +changed_files_status=$(run_get_pr_changed_file_details) +changed_files_status=$(echo "$changed_files_status" | grep "Cargo\.toml$" || true) +changed_files=$(echo "$changed_files_status" | awk '{print $NF}' || true) + +if [ -z "$changed_files" ]; then + cat >> "${cargo_deny_file}" << EOF + - run: echo "No Cargo.toml files to check" + shell: bash +EOF +fi + +for path in $changed_files +do + cat >> "${cargo_deny_file}" << EOF + + - name: ${path} + continue-on-error: true + shell: bash + run: | + pushd $(dirname ${path}) + cargo deny check + popd +EOF +done diff --git a/.github/cargo-deny-composite-action/cargo-deny-skeleton.yaml.in b/.github/cargo-deny-composite-action/cargo-deny-skeleton.yaml.in new file mode 100644 index 0000000000..e48d1f6c86 --- /dev/null +++ b/.github/cargo-deny-composite-action/cargo-deny-skeleton.yaml.in @@ -0,0 +1,30 @@ +# +# Copyright (c) 2022 Red Hat +# +# SPDX-License-Identifier: Apache-2.0 +# + +name: 'Cargo Crates Check' +description: 'Checks every Cargo.toml file using cargo-deny' + +env: + CARGO_TERM_COLOR: always + +runs: + using: "composite" + steps: + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: true + + - name: Cache + uses: Swatinem/rust-cache@v2 + + - name: Install Cargo deny + shell: bash + run: | + which cargo + cargo install --locked cargo-deny || true diff --git a/.github/workflows/cargo-deny-runner.yaml b/.github/workflows/cargo-deny-runner.yaml new file mode 100644 index 0000000000..5d6dfeb6cf --- /dev/null +++ b/.github/workflows/cargo-deny-runner.yaml @@ -0,0 +1,19 @@ +name: Cargo Crates Check Runner +on: [pull_request] +jobs: + cargo-deny-runner: + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + uses: actions/checkout@v3 + - name: Generate Action + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + run: bash cargo-deny-generator.sh + working-directory: ./.github/cargo-deny-composite-action/ + env: + GOPATH: ${{ runner.workspace }}/kata-containers + - name: Run Action + if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }} + uses: ./.github/cargo-deny-composite-action diff --git a/ci/lib.sh b/ci/lib.sh index 3cb2c158f6..29b640a6ae 100644 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -54,3 +54,13 @@ run_docs_url_alive_check() git fetch -a bash "$tests_repo_dir/.ci/static-checks.sh" --docs --all "github.com/kata-containers/kata-containers" } + +run_get_pr_changed_file_details() +{ + clone_tests_repo + # Make sure we have the targeting branch + git remote set-branches --add origin "${branch}" + git fetch -a + source "$tests_repo_dir/.ci/lib.sh" + get_pr_changed_file_details +} diff --git a/deny.toml b/deny.toml new file mode 100644 index 0000000000..7c97ec4c7e --- /dev/null +++ b/deny.toml @@ -0,0 +1,33 @@ +targets = [ + { triple = "x86_64-apple-darwin" }, + { triple = "x86_64-unknown-linux-gnu" }, + { triple = "x86_64-unknown-linux-musl" }, +] + +[advisories] +vulnerability = "deny" +unsound = "deny" +unmaintained = "deny" +ignore = ["RUSTSEC-2020-0071"] + +[bans] +multiple-versions = "allow" +deny = [ + { name = "cmake" }, + { name = "openssl-sys" }, +] + +[licenses] +unlicensed = "deny" +allow-osi-fsf-free = "neither" +copyleft = "allow" +# We want really high confidence when inferring licenses from text +confidence-threshold = 0.93 +allow = ["0BSD", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "CC0-1.0", "ISC", "MIT", "MPL-2.0"] +private = { ignore = true} + +exceptions = [] + +[sources] +unknown-registry = "allow" +unknown-git = "allow" From 7914da72c9c99bd993a8d01f69dfe5fd35e27bfa Mon Sep 17 00:00:00 2001 From: Derek Lee Date: Mon, 1 Aug 2022 11:14:41 -0700 Subject: [PATCH 06/26] cargo.tomls: Added Apache 2.0 to cargo.tomls One of the checks done by cargo-deny is ensuring all crates have a valid license. As the rust programs import each other, cargo.toml files without licenses trigger the check. While I could disable this check this would be bad practice. This adds an Apache-2.0 license in the Cargo.toml files. Some of these files already had a header comment saying it is an Apache license. As the entire project itself is under an Apache-2.0 license, I assumed all individual components would also be covered under that license. Signed-off-by: Derek Lee --- src/agent/Cargo.toml | 1 + src/agent/rustjail/Cargo.toml | 1 + src/agent/vsock-exporter/Cargo.toml | 1 + src/libs/logging/Cargo.toml | 1 + src/libs/oci/Cargo.toml | 1 + src/libs/protocols/Cargo.toml | 1 + src/runtime-rs/crates/agent/Cargo.toml | 1 + src/runtime-rs/crates/hypervisor/Cargo.toml | 1 + src/runtime-rs/crates/persist/Cargo.toml | 1 + src/runtime-rs/crates/resource/Cargo.toml | 1 + src/runtime-rs/crates/runtimes/Cargo.toml | 1 + src/runtime-rs/crates/runtimes/common/Cargo.toml | 1 + src/runtime-rs/crates/runtimes/virt_container/Cargo.toml | 1 + src/runtime-rs/crates/service/Cargo.toml | 1 + src/runtime-rs/tests/utils/Cargo.toml | 1 + src/tools/agent-ctl/Cargo.toml | 1 + src/tools/trace-forwarder/Cargo.toml | 1 + 17 files changed, 17 insertions(+) diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 96bb189c77..5e045e6340 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -3,6 +3,7 @@ name = "kata-agent" version = "0.1.0" authors = ["The Kata Containers community "] edition = "2018" +license = "Apache-2.0" [dependencies] oci = { path = "../libs/oci" } diff --git a/src/agent/rustjail/Cargo.toml b/src/agent/rustjail/Cargo.toml index 9daad2e666..b8cdb90299 100644 --- a/src/agent/rustjail/Cargo.toml +++ b/src/agent/rustjail/Cargo.toml @@ -3,6 +3,7 @@ name = "rustjail" version = "0.1.0" authors = ["The Kata Containers community "] edition = "2018" +license = "Apache-2.0" [dependencies] serde = "1.0.91" diff --git a/src/agent/vsock-exporter/Cargo.toml b/src/agent/vsock-exporter/Cargo.toml index f9f63b5c4b..0cdf0b91d2 100644 --- a/src/agent/vsock-exporter/Cargo.toml +++ b/src/agent/vsock-exporter/Cargo.toml @@ -3,6 +3,7 @@ name = "vsock-exporter" version = "0.1.0" authors = ["James O. D. Hunt "] edition = "2018" +license = "Apache-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/src/libs/logging/Cargo.toml b/src/libs/logging/Cargo.toml index 3457072bc6..c7cac4d7b5 100644 --- a/src/libs/logging/Cargo.toml +++ b/src/libs/logging/Cargo.toml @@ -3,6 +3,7 @@ name = "logging" version = "0.1.0" authors = ["The Kata Containers community "] edition = "2018" +license = "Apache-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/src/libs/oci/Cargo.toml b/src/libs/oci/Cargo.toml index dde7b9915c..8c08705a3d 100644 --- a/src/libs/oci/Cargo.toml +++ b/src/libs/oci/Cargo.toml @@ -3,6 +3,7 @@ name = "oci" version = "0.1.0" authors = ["The Kata Containers community "] edition = "2018" +license = "Apache-2.0" [dependencies] serde = "1.0.131" diff --git a/src/libs/protocols/Cargo.toml b/src/libs/protocols/Cargo.toml index 6853e9c259..03b9c8b3d5 100644 --- a/src/libs/protocols/Cargo.toml +++ b/src/libs/protocols/Cargo.toml @@ -3,6 +3,7 @@ name = "protocols" version = "0.1.0" authors = ["The Kata Containers community "] edition = "2018" +license = "Apache-2.0" [features] default = [] diff --git a/src/runtime-rs/crates/agent/Cargo.toml b/src/runtime-rs/crates/agent/Cargo.toml index c5febe43d7..69dd2b753b 100644 --- a/src/runtime-rs/crates/agent/Cargo.toml +++ b/src/runtime-rs/crates/agent/Cargo.toml @@ -3,6 +3,7 @@ name = "agent" version = "0.1.0" authors = ["The Kata Containers community "] edition = "2018" +license = "Apache-2.0" [dev-dependencies] futures = "0.1.27" diff --git a/src/runtime-rs/crates/hypervisor/Cargo.toml b/src/runtime-rs/crates/hypervisor/Cargo.toml index 4227de663c..56a27ecb46 100644 --- a/src/runtime-rs/crates/hypervisor/Cargo.toml +++ b/src/runtime-rs/crates/hypervisor/Cargo.toml @@ -3,6 +3,7 @@ name = "hypervisor" version = "0.1.0" authors = ["The Kata Containers community "] edition = "2018" +license = "Apache-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/src/runtime-rs/crates/persist/Cargo.toml b/src/runtime-rs/crates/persist/Cargo.toml index e0c6e5b1ff..4a5a32bfd9 100644 --- a/src/runtime-rs/crates/persist/Cargo.toml +++ b/src/runtime-rs/crates/persist/Cargo.toml @@ -3,6 +3,7 @@ name = "persist" version = "0.1.0" authors = ["The Kata Containers community "] edition = "2018" +license = "Apache-2.0" [dependencies] async-trait = "0.1.48" diff --git a/src/runtime-rs/crates/resource/Cargo.toml b/src/runtime-rs/crates/resource/Cargo.toml index 408baf522b..754ec800b2 100644 --- a/src/runtime-rs/crates/resource/Cargo.toml +++ b/src/runtime-rs/crates/resource/Cargo.toml @@ -3,6 +3,7 @@ name = "resource" version = "0.1.0" authors = ["The Kata Containers community "] edition = "2018" +license = "Apache-2.0" [dependencies] anyhow = "^1.0" diff --git a/src/runtime-rs/crates/runtimes/Cargo.toml b/src/runtime-rs/crates/runtimes/Cargo.toml index 8d7630b306..3347871fb2 100644 --- a/src/runtime-rs/crates/runtimes/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/Cargo.toml @@ -3,6 +3,7 @@ name = "runtimes" version = "0.1.0" authors = ["The Kata Containers community "] edition = "2018" +license = "Apache-2.0" [dependencies] anyhow = "^1.0" diff --git a/src/runtime-rs/crates/runtimes/common/Cargo.toml b/src/runtime-rs/crates/runtimes/common/Cargo.toml index ce52f5b772..00eb64825d 100644 --- a/src/runtime-rs/crates/runtimes/common/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/common/Cargo.toml @@ -3,6 +3,7 @@ name = "common" version = "0.1.0" authors = ["The Kata Containers community "] edition = "2018" +license = "Apache-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml b/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml index 0abe1b61b0..0e3fbdc60a 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml +++ b/src/runtime-rs/crates/runtimes/virt_container/Cargo.toml @@ -3,6 +3,7 @@ name = "virt_container" version = "0.1.0" authors = ["The Kata Containers community "] edition = "2018" +license = "Apache-2.0" [dependencies] anyhow = "^1.0" diff --git a/src/runtime-rs/crates/service/Cargo.toml b/src/runtime-rs/crates/service/Cargo.toml index 3b361a01cf..61fc3fb03f 100644 --- a/src/runtime-rs/crates/service/Cargo.toml +++ b/src/runtime-rs/crates/service/Cargo.toml @@ -3,6 +3,7 @@ name = "service" version = "0.1.0" authors = ["The Kata Containers community "] edition = "2018" +license = "Apache-2.0" [dependencies] anyhow = "^1.0" diff --git a/src/runtime-rs/tests/utils/Cargo.toml b/src/runtime-rs/tests/utils/Cargo.toml index 7317b7f0ff..c4fc094719 100644 --- a/src/runtime-rs/tests/utils/Cargo.toml +++ b/src/runtime-rs/tests/utils/Cargo.toml @@ -3,6 +3,7 @@ name = "tests_utils" version = "0.1.0" edition = "2018" description = "This crate is used to share code among tests" +license = "Apache-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/src/tools/agent-ctl/Cargo.toml b/src/tools/agent-ctl/Cargo.toml index 8d2f93b847..25852d83ba 100644 --- a/src/tools/agent-ctl/Cargo.toml +++ b/src/tools/agent-ctl/Cargo.toml @@ -8,6 +8,7 @@ name = "kata-agent-ctl" version = "0.0.1" authors = ["The Kata Containers community "] edition = "2018" +license = "Apache-2.0" [dependencies] protocols = { path = "../../libs/protocols", features = ["with-serde"] } diff --git a/src/tools/trace-forwarder/Cargo.toml b/src/tools/trace-forwarder/Cargo.toml index 2579ae5b6c..fc0b69631e 100644 --- a/src/tools/trace-forwarder/Cargo.toml +++ b/src/tools/trace-forwarder/Cargo.toml @@ -8,6 +8,7 @@ name = "kata-trace-forwarder" version = "0.0.1" authors = ["The Kata Containers community "] edition = "2018" +license = "Apache-2.0" [dependencies] futures = "0.3.15" From aa581f4b28ed45921d74718a9423b78009c94a1b Mon Sep 17 00:00:00 2001 From: Derek Lee Date: Mon, 1 Aug 2022 11:24:45 -0700 Subject: [PATCH 07/26] cargo.toml: Add oci to src/libs workplace Adds oci under the src/libs workplace. oci shares a Cargo.lock file with the rest of src/libs but was not listed as a member of the workspace. There is no clear reason why it is not included in the workspace, so adding it so cargo-deny stop complaining Signed-off-by: Derek Lee --- src/libs/Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libs/Cargo.toml b/src/libs/Cargo.toml index b173ae7b77..62372e816a 100644 --- a/src/libs/Cargo.toml +++ b/src/libs/Cargo.toml @@ -1,11 +1,11 @@ [workspace] members = [ - "logging", - "kata-types", "kata-sys-util", - "safe-path", - "protocols", + "kata-types", + "logging", "oci", + "protocols", + "safe-path", "test-utils", ] resolver = "2" From 52bbc3a4b05e990f3e5e490592c2e0ed3e207cca Mon Sep 17 00:00:00 2001 From: Derek Lee Date: Tue, 30 Aug 2022 10:00:18 -0700 Subject: [PATCH 08/26] cargo.lock: update crates to comply with checks Updates versions of crossbeam-channel because 0.52.0 is a yanked package (creators mark version as not for release except as a dependency for another package) Updates chrono to use >0.42.0 to avoid: https://rustsec.org/advisories/RUSTSEC-2020-0159 Updates lz4-sys. Signed-off-by: Derek Lee --- src/agent/Cargo.lock | 30 +++++++- src/libs/Cargo.lock | 88 +++++++++++++++++++--- src/runtime-rs/Cargo.lock | 44 +++++++++-- src/tools/agent-ctl/Cargo.lock | 105 ++++++++++++++++++++++++++- src/tools/trace-forwarder/Cargo.lock | 7 +- 5 files changed, 249 insertions(+), 25 deletions(-) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 27e7e3953c..498a6af2f2 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -17,6 +17,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -180,14 +189,16 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.19" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" dependencies = [ - "libc", + "iana-time-zone", + "js-sys", "num-integer", "num-traits", "time 0.1.44", + "wasm-bindgen", "winapi", ] @@ -529,6 +540,19 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "iana-time-zone" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad2bfd338099682614d3ee3fe0cd72e0b6a41ca6a87f6a74a3bd593c91650501" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "js-sys", + "wasm-bindgen", + "winapi", +] + [[package]] name = "indexmap" version = "1.9.1" diff --git a/src/libs/Cargo.lock b/src/libs/Cargo.lock index 380f025135..3822058e59 100644 --- a/src/libs/Cargo.lock +++ b/src/libs/Cargo.lock @@ -46,6 +46,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "bumpalo" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" + [[package]] name = "byte-unit" version = "3.1.4" @@ -100,14 +106,15 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +checksum = "6127248204b9aba09a362f6c930ef6a78f2c1b2215f8a7b398c06e1083f17af0" dependencies = [ - "libc", + "js-sys", "num-integer", "num-traits", "time", + "wasm-bindgen", "winapi", ] @@ -119,9 +126,9 @@ checksum = "2382f75942f4b3be3690fe4f86365e9c853c1587d6ee58212cebf6e2a9ccd101" [[package]] name = "crossbeam-channel" -version = "0.5.2" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" dependencies = [ "cfg-if", "crossbeam-utils", @@ -364,6 +371,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +[[package]] +name = "js-sys" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "kata-sys-util" version = "0.1.0" @@ -821,9 +837,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.4" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" dependencies = [ "aho-corasick", "memchr", @@ -832,9 +848,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.25" +version = "0.6.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" [[package]] name = "remove_dir_all" @@ -1203,6 +1219,60 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be" + [[package]] name = "which" version = "4.2.5" diff --git a/src/runtime-rs/Cargo.lock b/src/runtime-rs/Cargo.lock index 428b8047d0..a37eb41874 100644 --- a/src/runtime-rs/Cargo.lock +++ b/src/runtime-rs/Cargo.lock @@ -69,6 +69,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.57" @@ -400,14 +409,16 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.19" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" dependencies = [ - "libc", + "iana-time-zone", + "js-sys", "num-integer", "num-traits", "time 0.1.43", + "wasm-bindgen", "winapi", ] @@ -476,6 +487,12 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "cpuid-bool" version = "0.1.2" @@ -754,7 +771,7 @@ version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f82432ae94d42f160b6e17389d6e1c1eee29827b99ad32d35a0a96bb98bedb5" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.2.3", "libc", ] @@ -912,7 +929,7 @@ dependencies = [ "arc-swap 1.5.0", "bitflags", "caps", - "core-foundation-sys", + "core-foundation-sys 0.2.3", "diskarbitration-sys", "lazy_static", "libc", @@ -1183,6 +1200,19 @@ dependencies = [ "vmm-sys-util", ] +[[package]] +name = "iana-time-zone" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad2bfd338099682614d3ee3fe0cd72e0b6a41ca6a87f6a74a3bd593c91650501" +dependencies = [ + "android_system_properties", + "core-foundation-sys 0.8.3", + "js-sys", + "wasm-bindgen", + "winapi", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1418,9 +1448,9 @@ dependencies = [ [[package]] name = "lz4-sys" -version = "1.9.3" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7be8908e2ed6f31c02db8a9fa962f03e36c53fbfde437363eae3306b85d7e17" +checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" dependencies = [ "cc", "libc", diff --git a/src/tools/agent-ctl/Cargo.lock b/src/tools/agent-ctl/Cargo.lock index 608c519d28..414ce3c4a6 100644 --- a/src/tools/agent-ctl/Cargo.lock +++ b/src/tools/agent-ctl/Cargo.lock @@ -11,6 +11,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -66,6 +75,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "bumpalo" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" + [[package]] name = "byteorder" version = "1.4.3" @@ -131,14 +146,16 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.19" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" dependencies = [ - "libc", + "iana-time-zone", + "js-sys", "num-integer", "num-traits", "time", + "wasm-bindgen", "winapi", ] @@ -157,6 +174,12 @@ dependencies = [ "vec_map", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "crossbeam-channel" version = "0.5.1" @@ -362,6 +385,19 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "iana-time-zone" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad2bfd338099682614d3ee3fe0cd72e0b6a41ca6a87f6a74a3bd593c91650501" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "js-sys", + "wasm-bindgen", + "winapi", +] + [[package]] name = "indexmap" version = "1.7.0" @@ -409,6 +445,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +[[package]] +name = "js-sys" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "kata-agent-ctl" version = "0.0.1" @@ -1160,6 +1205,60 @@ version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +[[package]] +name = "wasm-bindgen" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be" + [[package]] name = "which" version = "4.2.2" diff --git a/src/tools/trace-forwarder/Cargo.lock b/src/tools/trace-forwarder/Cargo.lock index 68e3ec4cdf..ce1f994d26 100644 --- a/src/tools/trace-forwarder/Cargo.lock +++ b/src/tools/trace-forwarder/Cargo.lock @@ -92,14 +92,15 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +checksum = "6127248204b9aba09a362f6c930ef6a78f2c1b2215f8a7b398c06e1083f17af0" dependencies = [ - "libc", + "js-sys", "num-integer", "num-traits", "time", + "wasm-bindgen", "winapi", ] From 0ab49b233eec02ab0cc4b59da239a132001607bc Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Tue, 30 Aug 2022 12:59:10 -0700 Subject: [PATCH 09/26] release: Kata Containers 3.0.0-alpha1 - Initrd fixes for ubuntu systemd - kernel: Add CONFIG_CGROUP_HUGETLB=y as part of the cgroup fragments - Fix kata-deploy to work on CI context - github-actions: Auto-backporting - runtime-rs: add support for core scheduling - ci: Use versions.yaml for the libseccomp - runk: Add cli message for init command - agent: add some logs for mount operation - Use iouring for qemu block devices - logging: Replace nix::Error::EINVAL with more descriptive msgs - kata-deploy: fix threading conflicts - kernel: Ignore CONFIG_SPECULATION_MITIGATIONS for older kernels - runtime-rs: support loading kernel modules in guest vm - TDX: Get TDX working again with Cloud Hypervisor + a minor change on QEMU's code - runk: Move delete logic to libcontainer - runtime: cri-o annotations have been moved to podman - Fix depbot reported rust crates dependency security issues - UT: test_load_kernel_module needs root - enable vmx for vm factory - runk: add pause/resume commands - kernel: upgrade guest kernel support to 5.19 - Drop-in cfg files support in runtime-rs - agent: do some rollback works if case of do_create_container failed - network: Fix error message for setting hardware address on TAP interface - Upgrade to Cloud Hypervisor v26.0 - runtime: tracing: End root span at end of trace - ci: Update libseccomp version - dep: update nix dependency - Updated the link target of CRI-O - libs/test-utils: share test code by create a new crate dc32c4622 osbuilder: fix ubuntu initrd /dev/ttyS0 hang cc5f91dac osbuilder: add systemd symlinks for kata-agent c08a8631e agent: add some logs for mount operation 0a6f0174f kernel: Ignore CONFIG_SPECULATION_MITIGATIONS for older kernels 6cf16c4f7 agent-ctl: fix clippy error 4b57c04c3 runtime-rs: support loading kernel modules in guest vm dc90eae17 qemu: Drop unnecessary `tdx_guest` kernel parameter d4b67613f clh: Use HVC console with TDX c0cb3cd4d clh: Avoid crashing when memory hotplug is not allowed 9f0a57c0e clh: Increase API and SandboxStop timeouts for TDX b535bac9c runk: Add cli message for init command c142fa254 clh: Lift the sharedFS restriction used with TDX bdf8a57bd runk: Move delete logic to libcontainer a06d819b2 runtime: cri-o annotations have been moved to podman ffd1c1ff4 agent-ctl/trace-forwarder: udpate thread_local dependency 69080d76d agent/runk: update regex dependency e0ec09039 runtime-rs: update async-std dependency 763ceeb7b logging: Replace nix::Error::EINVAL with more descriptive msgs 4ee2b99e1 kata-deploy: fix threading conflicts 731d39df4 kernel: Add CONFIG_CGROUP_HUGETLB=y as part of the cgroup fragments 96d903734 github-actions: Auto-backporting a6fbaac1b runk: add pause/resume commands 8e201501e kernel: fix for set_kmem_limit error 00aadfe20 kernel: SEV guest kernel upgrade to 5.19.2 0d9d8d63e kernel: upgrade guest kernel support to 5.19.2 57bd3f42d runtime-rs: plug drop-in decoding into config-loading code 87b97b699 runtime-rs: add filesystem-related part of drop-in handling cf785a1a2 runtime-rs: add core toml::Value tree merging 92f7d6bf8 ci: Use versions.yaml for the libseccomp f508c2909 runtime: constify splitIrqChipMachineOptions 2b0587db9 runtime: VMX is migratible in vm factory case fa09f0ec8 runtime: remove qemuPaths 326f1cc77 agent: enrich some error code path 4f53e010b agent: skip test_load_kernel_module if non-root 3a597c274 runtime: clh: Use the new 'payload' interface 16baecc5b runtime: clh: Re-generate the client code 50ea07183 versions: Upgrade to Cloud Hypervisor v26.0 f7d41e98c kata-deploy: export CI in the build container 4f90e3c87 kata-deploy: add dockerbuild/install_yq.sh to gitignore 8ff5c10ac network: Fix error message for setting hardware address on TAP interface 338c28295 dep: update nix dependency 78231a36e ci: Update libseccomp version 34746496b libs/test-utils: share test code by create a new crate 3829ab809 docs: Update CRI-O target link fcc1e0c61 runtime: tracing: End root span at end of trace c1e3b8f40 govmm: Refactor qmp functions for adding block device 598884f37 govmm: Refactor code to get rid of redundant code 00860a7e4 qmp: Pass aio backend while adding block device e1b49d758 config: Add block aio as a supported annotation ed0f1d0b3 config: Add "block_device_aio" as a config option for qemu b6cd2348f govmm: Add io_uring as AIO type 81cdaf077 govmm: Correct documentation for Linux aio. a355812e0 runtime-rs: fixed bug on core-sched error handling 591dfa4fe runtime-rs: add support for core scheduling 09672eb2d agent: do some rollback works if case of do_create_container failed Signed-off-by: Archana Shinde --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6076f48d63..df7f8449cf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.0-alpha0 +3.0.0-alpha1 From e0142db24f8c8755c3a407a64e25f9c70bb04a35 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Tue, 9 Aug 2022 14:56:10 -0700 Subject: [PATCH 10/26] hypervisor: Add GetTotalMemoryMB to interface It'll be useful to get the total memory provided to the guest (hotplugged + coldplugged). We'll use this information when calcualting how much memory we can add at a time when utilizing ACPI hotplug. Signed-off-by: Eric Ernst --- src/runtime/virtcontainers/acrn.go | 4 ++++ src/runtime/virtcontainers/clh.go | 10 ++++++++++ src/runtime/virtcontainers/fc.go | 4 ++++ src/runtime/virtcontainers/hypervisor.go | 1 + src/runtime/virtcontainers/mock_hypervisor.go | 3 +++ src/runtime/virtcontainers/qemu.go | 6 +++++- 6 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/runtime/virtcontainers/acrn.go b/src/runtime/virtcontainers/acrn.go index 1c3ebc1476..008b3bd97c 100644 --- a/src/runtime/virtcontainers/acrn.go +++ b/src/runtime/virtcontainers/acrn.go @@ -667,6 +667,10 @@ func (a *Acrn) GetThreadIDs(ctx context.Context) (VcpuThreadIDs, error) { return VcpuThreadIDs{}, nil } +func (a *Acrn) GetTotalMemoryMB(ctx context.Context) uint32 { + return a.config.MemorySize +} + func (a *Acrn) ResizeMemory(ctx context.Context, reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, MemoryDevice, error) { return 0, MemoryDevice{}, nil } diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 3a02a645a9..d93ceed675 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -1598,6 +1598,16 @@ func (clh *cloudHypervisor) cleanupVM(force bool) error { return nil } +func (clh *cloudHypervisor) GetTotalMemoryMB(ctx context.Context) uint32 { + vminfo, err := clh.vmInfo() + if err != nil { + clh.Logger().WithError(err).Error("failed to get vminfo") + return 0 + } + + return uint32(vminfo.GetMemoryActualSize() >> utils.MibToBytesShift) +} + // vmInfo ask to hypervisor for current VM status func (clh *cloudHypervisor) vmInfo() (chclient.VmInfo, error) { cl := clh.client() diff --git a/src/runtime/virtcontainers/fc.go b/src/runtime/virtcontainers/fc.go index 703e6e88b5..f81cc319cf 100644 --- a/src/runtime/virtcontainers/fc.go +++ b/src/runtime/virtcontainers/fc.go @@ -1165,6 +1165,10 @@ func (fc *firecracker) HypervisorConfig() HypervisorConfig { return fc.config } +func (fc *firecracker) GetTotalMemoryMB(ctx context.Context) uint32 { + return fc.config.MemorySize +} + func (fc *firecracker) ResizeMemory(ctx context.Context, reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, MemoryDevice, error) { return 0, MemoryDevice{}, nil } diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 49b658db31..0e7b4785bb 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -922,6 +922,7 @@ type Hypervisor interface { HotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType DeviceType) (interface{}, error) ResizeMemory(ctx context.Context, memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, MemoryDevice, error) ResizeVCPUs(ctx context.Context, vcpus uint32) (uint32, uint32, error) + GetTotalMemoryMB(ctx context.Context) uint32 GetVMConsole(ctx context.Context, sandboxID string) (string, string, error) Disconnect(ctx context.Context) Capabilities(ctx context.Context) types.Capabilities diff --git a/src/runtime/virtcontainers/mock_hypervisor.go b/src/runtime/virtcontainers/mock_hypervisor.go index f4a0b934ec..19b818dff8 100644 --- a/src/runtime/virtcontainers/mock_hypervisor.go +++ b/src/runtime/virtcontainers/mock_hypervisor.go @@ -98,6 +98,9 @@ func (m *mockHypervisor) ResizeVCPUs(ctx context.Context, cpus uint32) (uint32, return 0, 0, nil } +func (m *mockHypervisor) GetTotalMemoryMB(ctx context.Context) uint32 { + return 0 +} func (m *mockHypervisor) Disconnect(ctx context.Context) { } diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 56bd5c3899..6ef2310f41 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -2166,6 +2166,10 @@ func (q *qemu) Disconnect(ctx context.Context) { q.qmpShutdown() } +func (q *qemu) GetTotalMemoryMB(ctx context.Context) uint32 { + return q.config.MemorySize + uint32(q.state.HotpluggedMemory) +} + // ResizeMemory gets a request to update the VM memory to reqMemMB // Memory update is managed with two approaches // Add memory to VM: @@ -2179,7 +2183,7 @@ func (q *qemu) Disconnect(ctx context.Context) { // A longer term solution is evaluate solutions like virtio-mem func (q *qemu) ResizeMemory(ctx context.Context, reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, MemoryDevice, error) { - currentMemory := q.config.MemorySize + uint32(q.state.HotpluggedMemory) + currentMemory := q.GetTotalMemoryMB(ctx) if err := q.qmpSetup(); err != nil { return 0, MemoryDevice{}, err } From 566656b085787b5927129f723e4bd8febbfc250b Mon Sep 17 00:00:00 2001 From: Ryan Savino Date: Wed, 31 Aug 2022 10:02:53 -0500 Subject: [PATCH 11/26] gperf: point URL to mirror site gperf download fails intermittently. Changing to mirror site will hopefully increase download reliability. Fixes: #5057 Signed-Off-By: Ryan Savino --- versions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.yaml b/versions.yaml index 491ed6f06b..eaedb92006 100644 --- a/versions.yaml +++ b/versions.yaml @@ -209,7 +209,7 @@ externals: gperf: description: "GNU gperf is a perfect hash function generator" - url: "https://ftp.gnu.org/gnu/gperf" + url: "https://ftpmirror.gnu.org/gnu/gperf" version: "3.1" kubernetes: From f390c122f065cfca3c4be057f6560544d3cf650d Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Tue, 9 Aug 2022 15:36:09 -0700 Subject: [PATCH 12/26] sandbox: don't hotplug too much memory at once If we're using ACPI hotplug for memory, there's a limitation on the amount of memory which can be hotplugged at a single time. During hotplug, we'll allocate memory for the memmap for each page, resulting in a 64 byte per 4KiB page allocation. As an example, hotplugging 12GiB of memory requires ~192 MiB of *free* memory, which is about the limit we should expect for an idle 256 MiB guest (conservative heuristic of 75% of provided memory). From experimentation, at pod creation time we can reliably add 48 times what is provided to the guest. (a factor of 48 results in using 75% of provided memory for hotplug). Using prior example of a guest with 256Mi RAM, 256 Mi * 48 = 12 Gi; 12GiB is upper end of what we should expect can be hotplugged successfully into the guest. Note: It isn't expected that we'll need to hotplug large amounts of RAM after workloads have already started -- container additions are expected to occur first in pod lifecycle. Based on this, we expect that provided memory should be freely available for hotplug. If virtio-mem is being utilized, there isn't such a limitation - we can hotplug the max allowed memory at a single time. Fixes: #4847 Signed-off-by: Eric Ernst --- src/runtime/virtcontainers/sandbox.go | 64 +++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index e691ea1dee..e4a16983ed 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -77,6 +77,14 @@ const ( // Restricted permission for shared directory managed by virtiofs sharedDirMode = os.FileMode(0700) | os.ModeDir + + // hotplug factor indicates how much memory can be hotplugged relative to the amount of + // RAM provided to the guest. This is a conservative heuristic based on needing 64 bytes per + // 4KiB page of hotplugged memory. + // + // As an example: 12 GiB hotplugged -> 3 Mi pages -> 192 MiBytes overhead (3Mi x 64B). + // This is approximately what should be free in a relatively unloaded 256 MiB guest (75% of available memory). So, 256 Mi x 48 => 12 Gi + acpiMemoryHotplugFactor = 48 ) var ( @@ -2012,9 +2020,60 @@ func (s *Sandbox) updateResources(ctx context.Context) error { } s.Logger().Debugf("Sandbox CPUs: %d", newCPUs) - // Update Memory - s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory") + // Update Memory -- + // If we're using ACPI hotplug for memory, there's a limitation on the amount of memory which can be hotplugged at a single time. + // We must have enough free memory in the guest kernel to cover 64bytes per (4KiB) page of memory added for mem_map. + // See https://github.com/kata-containers/kata-containers/issues/4847 for more details. + // For a typical pod lifecycle, we expect that each container is added when we start the workloads. Based on this, we'll "assume" that majority + // of the guest memory is readily available. From experimentation, we see that we can add approximately 48 times what is already provided to + // the guest workload. For example, a 256 MiB guest should be able to accommodate hotplugging 12 GiB of memory. + // + // If virtio-mem is being used, there isn't such a limitation - we can hotplug the maximum allowed memory at a single time. + // newMemoryMB := uint32(sandboxMemoryByte >> utils.MibToBytesShift) + finalMemoryMB := newMemoryMB + + hconfig := s.hypervisor.HypervisorConfig() + + for { + currentMemoryMB := s.hypervisor.GetTotalMemoryMB(ctx) + + maxhotPluggableMemoryMB := currentMemoryMB * acpiMemoryHotplugFactor + + // In the case of virtio-mem, we don't have a restriction on how much can be hotplugged at + // a single time. As a result, the max hotpluggable is only limited by the maximum memory size + // of the guest. + if hconfig.VirtioMem { + maxhotPluggableMemoryMB = uint32(hconfig.DefaultMaxMemorySize) - currentMemoryMB + } + + deltaMB := int32(finalMemoryMB - currentMemoryMB) + + if deltaMB > int32(maxhotPluggableMemoryMB) { + s.Logger().Warnf("Large hotplug. Adding %d MB of %d total memory", maxhotPluggableMemoryMB, deltaMB) + newMemoryMB = currentMemoryMB + maxhotPluggableMemoryMB + } else { + newMemoryMB = finalMemoryMB + } + + // Add the memory to the guest and online the memory: + if err := s.updateMemory(ctx, newMemoryMB); err != nil { + return err + } + + if newMemoryMB == finalMemoryMB { + break + } + + } + + return nil + +} + +func (s *Sandbox) updateMemory(ctx context.Context, newMemoryMB uint32) error { + // online the memory: + s.Logger().WithField("memory-sandbox-size-mb", newMemoryMB).Debugf("Request to hypervisor to update memory") newMemory, updatedMemoryDevice, err := s.hypervisor.ResizeMemory(ctx, newMemoryMB, s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe) if err != nil { if err == noGuestMemHotplugErr { @@ -2034,7 +2093,6 @@ func (s *Sandbox) updateResources(ctx context.Context) error { if err := s.agent.onlineCPUMem(ctx, 0, false); err != nil { return err } - return nil } From 9997ab064a592c412e277dd1cf48bdc9e1119220 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Fri, 22 Jul 2022 19:35:34 -0700 Subject: [PATCH 13/26] sandbox_test: Add test to verify memory hotplug behavior Augment the mock hypervisor so that we can validate that ACPI memory hotplug is carried out as expected. We'll augment the number of memory slots in the hypervisor config each time the memory of the hypervisor is changed. In this way we can ensure that large memory hotplugs are broken up into appropriately sized pieces in the unit test. Signed-off-by: Eric Ernst --- src/runtime/virtcontainers/mock_hypervisor.go | 13 ++++++--- .../virtcontainers/mock_hypervisor_test.go | 2 +- src/runtime/virtcontainers/sandbox_test.go | 27 ++++++++++++------- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/runtime/virtcontainers/mock_hypervisor.go b/src/runtime/virtcontainers/mock_hypervisor.go index 19b818dff8..7d6da561fa 100644 --- a/src/runtime/virtcontainers/mock_hypervisor.go +++ b/src/runtime/virtcontainers/mock_hypervisor.go @@ -17,6 +17,7 @@ import ( var MockHybridVSockPath = "/tmp/kata-mock-hybrid-vsock.socket" type mockHypervisor struct { + config HypervisorConfig mockPid int } @@ -27,10 +28,11 @@ func (m *mockHypervisor) Capabilities(ctx context.Context) types.Capabilities { } func (m *mockHypervisor) HypervisorConfig() HypervisorConfig { - return HypervisorConfig{} + return m.config } func (m *mockHypervisor) setConfig(config *HypervisorConfig) error { + m.config = *config return nil } @@ -38,7 +40,7 @@ func (m *mockHypervisor) CreateVM(ctx context.Context, id string, network Networ if err := m.setConfig(hypervisorConfig); err != nil { return err } - + m.config.MemSlots = 0 return nil } @@ -92,6 +94,11 @@ func (m *mockHypervisor) GetVMConsole(ctx context.Context, sandboxID string) (st } func (m *mockHypervisor) ResizeMemory(ctx context.Context, memMB uint32, memorySectionSizeMB uint32, probe bool) (uint32, MemoryDevice, error) { + if m.config.MemorySize != memMB { + // For testing, we'll use MemSlots to track how many times we resized memory + m.config.MemSlots += 1 + m.config.MemorySize = memMB + } return 0, MemoryDevice{}, nil } func (m *mockHypervisor) ResizeVCPUs(ctx context.Context, cpus uint32) (uint32, uint32, error) { @@ -99,7 +106,7 @@ func (m *mockHypervisor) ResizeVCPUs(ctx context.Context, cpus uint32) (uint32, } func (m *mockHypervisor) GetTotalMemoryMB(ctx context.Context) uint32 { - return 0 + return m.config.MemorySize } func (m *mockHypervisor) Disconnect(ctx context.Context) { } diff --git a/src/runtime/virtcontainers/mock_hypervisor_test.go b/src/runtime/virtcontainers/mock_hypervisor_test.go index 0159a993dd..ba4435f13b 100644 --- a/src/runtime/virtcontainers/mock_hypervisor_test.go +++ b/src/runtime/virtcontainers/mock_hypervisor_test.go @@ -14,7 +14,7 @@ import ( ) func TestMockHypervisorCreateVM(t *testing.T) { - var m *mockHypervisor + m := &mockHypervisor{} assert := assert.New(t) sandbox := &Sandbox{ diff --git a/src/runtime/virtcontainers/sandbox_test.go b/src/runtime/virtcontainers/sandbox_test.go index 331094ee39..59ed24c1aa 100644 --- a/src/runtime/virtcontainers/sandbox_test.go +++ b/src/runtime/virtcontainers/sandbox_test.go @@ -41,6 +41,7 @@ func newHypervisorConfig(kernelParams []Param, hParams []Param) HypervisorConfig HypervisorPath: filepath.Join(testDir, testHypervisor), KernelParams: kernelParams, HypervisorParams: hParams, + MemorySize: 1, } } @@ -1360,7 +1361,6 @@ func TestSandboxUpdateResources(t *testing.T) { contConfig1 := newTestContainerConfigNoop("cont-00001") contConfig2 := newTestContainerConfigNoop("cont-00002") hConfig := newHypervisorConfig(nil, nil) - defer cleanUp() // create a sandbox s, err := testCreateSandbox(t, @@ -1370,28 +1370,37 @@ func TestSandboxUpdateResources(t *testing.T) { NetworkConfig{}, []ContainerConfig{contConfig1, contConfig2}, nil) - assert.NoError(t, err) + err = s.updateResources(context.Background()) assert.NoError(t, err) - containerMemLimit := int64(1000) + // For mock hypervisor, we MemSlots to be 0 since the memory wasn't changed. + assert.Equal(t, s.hypervisor.HypervisorConfig().MemSlots, uint32(0)) + + containerMemLimit := int64(4 * 1024 * 1024 * 1024) containerCPUPeriod := uint64(1000) containerCPUQouta := int64(5) - for _, c := range s.config.Containers { - c.Resources.Memory = &specs.LinuxMemory{ + for idx := range s.config.Containers { + s.config.Containers[idx].Resources.Memory = &specs.LinuxMemory{ Limit: new(int64), } - c.Resources.CPU = &specs.LinuxCPU{ + s.config.Containers[idx].Resources.CPU = &specs.LinuxCPU{ Period: new(uint64), Quota: new(int64), } - c.Resources.Memory.Limit = &containerMemLimit - c.Resources.CPU.Period = &containerCPUPeriod - c.Resources.CPU.Quota = &containerCPUQouta + s.config.Containers[idx].Resources.Memory.Limit = &containerMemLimit + s.config.Containers[idx].Resources.CPU.Period = &containerCPUPeriod + s.config.Containers[idx].Resources.CPU.Quota = &containerCPUQouta } err = s.updateResources(context.Background()) assert.NoError(t, err) + + // Since we're starting with a memory of 1 MB, we expect it to take 3 hotplugs to add 4GiB of memory when using ACPI hotplug: + // +48MB + // +2352MB + // +the remaining + assert.Equal(t, s.hypervisor.HypervisorConfig().MemSlots, uint32(3)) } func TestSandboxExperimentalFeature(t *testing.T) { From 749a6a248053b483f0a8de77fefd313a9f3bb615 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Thu, 1 Sep 2022 13:54:31 +0800 Subject: [PATCH 14/26] docs: Specify language in markdown for syntax highlight Specify language for code block in docs/Unit-Test-Advice.md for syntax highlight. Fixes: #5064 Signed-off-by: Bin Liu --- docs/Unit-Test-Advice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Unit-Test-Advice.md b/docs/Unit-Test-Advice.md index ea6caa031c..4bd4da5299 100644 --- a/docs/Unit-Test-Advice.md +++ b/docs/Unit-Test-Advice.md @@ -341,7 +341,7 @@ The main repository has the most comprehensive set of skip abilities. See: One method is to use the `nix` crate along with some custom macros: -``` +```rust #[cfg(test)] mod tests { #[allow(unused_macros)] From 41ec71169f9d323f97b43b6aba3732964fd047aa Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Thu, 1 Sep 2022 10:28:01 +0800 Subject: [PATCH 15/26] runtime-rs: split amend_spec function amend_spec do two works: - modify the spec - check if the pid namespace is enabled This make it confusable. So split it into two functions. Fixes: #5062 Signed-off-by: Bin Liu --- .../src/container_manager/container.rs | 116 +++++++++++------- 1 file changed, 72 insertions(+), 44 deletions(-) diff --git a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs index 2d414318bd..834c68b77c 100644 --- a/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs +++ b/src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs @@ -80,8 +80,9 @@ impl Container { let mut inner = self.inner.write().await; let toml_config = self.resource_manager.config().await; let config = &self.config; - let sandbox_pidns = amend_spec(&mut spec, toml_config.runtime.disable_guest_seccomp) - .context("load spec")?; + amend_spec(&mut spec, toml_config.runtime.disable_guest_seccomp).context("amend spec")?; + let sandbox_pidns = is_pid_namespace_enabled(&spec); + // handler rootfs let rootfs = self .resource_manager @@ -373,7 +374,7 @@ impl Container { } } -fn amend_spec(spec: &mut oci::Spec, disable_guest_seccomp: bool) -> Result { +fn amend_spec(spec: &mut oci::Spec, disable_guest_seccomp: bool) -> Result<()> { // hook should be done on host spec.hooks = None; @@ -401,33 +402,29 @@ fn amend_spec(spec: &mut oci::Spec, disable_guest_seccomp: bool) -> Result } linux.namespaces = ns; - - return Ok(handle_pid_namespace(&linux.namespaces)); } - Ok(false) + Ok(()) } -// handle_pid_namespace checks if Pid namespace for a container needs to be shared with its sandbox +// is_pid_namespace_enabled checks if Pid namespace for a container needs to be shared with its sandbox // pid namespace. -fn handle_pid_namespace(namespaces: &[oci::LinuxNamespace]) -> bool { - for n in namespaces.iter() { - match n.r#type.as_str() { - oci::PIDNAMESPACE => { - if !n.path.is_empty() { - return true; - } +fn is_pid_namespace_enabled(spec: &oci::Spec) -> bool { + if let Some(linux) = spec.linux.as_ref() { + for n in linux.namespaces.iter() { + if n.r#type.as_str() == oci::PIDNAMESPACE { + return !n.path.is_empty(); } - _ => continue, } } + false } #[cfg(test)] mod tests { use super::amend_spec; - use crate::container_manager::container::handle_pid_namespace; + use super::is_pid_namespace_enabled; #[test] fn test_amend_spec_disable_guest_seccomp() { let mut spec = oci::Spec { @@ -448,38 +445,69 @@ mod tests { amend_spec(&mut spec, true).unwrap(); assert!(spec.linux.as_ref().unwrap().seccomp.is_none()); } + #[test] - fn test_handle_pid_namespace() { - let namespaces = vec![ - oci::LinuxNamespace { - r#type: "pid".to_string(), - path: "".to_string(), + fn test_is_pid_namespace_enabled() { + struct TestData<'a> { + desc: &'a str, + namespaces: Vec, + result: bool, + } + + let tests = &[ + TestData { + desc: "no pid namespace", + namespaces: vec![oci::LinuxNamespace { + r#type: "network".to_string(), + path: "".to_string(), + }], + result: false, }, - oci::LinuxNamespace { - r#type: "network".to_string(), - path: "".to_string(), + TestData { + desc: "empty pid namespace path", + namespaces: vec![ + oci::LinuxNamespace { + r#type: "pid".to_string(), + path: "".to_string(), + }, + oci::LinuxNamespace { + r#type: "network".to_string(), + path: "".to_string(), + }, + ], + result: false, }, - oci::LinuxNamespace { - r#type: "ipc".to_string(), - path: "".to_string(), - }, - oci::LinuxNamespace { - r#type: "uts".to_string(), - path: "".to_string(), - }, - oci::LinuxNamespace { - r#type: "mount".to_string(), - path: "".to_string(), - }, - oci::LinuxNamespace { - r#type: "user".to_string(), - path: "".to_string(), - }, - oci::LinuxNamespace { - r#type: "cgroup".to_string(), - path: "".to_string(), + TestData { + desc: "pid namespace is set", + namespaces: vec![ + oci::LinuxNamespace { + r#type: "pid".to_string(), + path: "/some/path".to_string(), + }, + oci::LinuxNamespace { + r#type: "network".to_string(), + path: "".to_string(), + }, + ], + result: true, }, ]; - assert!(!handle_pid_namespace(&namespaces)); + + let mut spec = oci::Spec::default(); + + for (i, d) in tests.iter().enumerate() { + spec.linux = Some(oci::Linux { + namespaces: d.namespaces.clone(), + ..Default::default() + }); + + assert_eq!( + d.result, + is_pid_namespace_enabled(&spec), + "test[{}]: {:?}", + i, + d.desc + ); + } } } From e83b82131629185e5830bc594a958d72c8baba4f Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Thu, 1 Sep 2022 15:33:29 +0000 Subject: [PATCH 16/26] docs: Update url in the Developer Guide This PR updates the url for containerd in the Developer Guide. Fixes #5075 Signed-off-by: Gabriela Cervantes --- docs/Developer-Guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md index c1c2d62ab1..ed9bec4903 100644 --- a/docs/Developer-Guide.md +++ b/docs/Developer-Guide.md @@ -522,7 +522,7 @@ bash-4.2# exit exit ``` -`kata-runtime exec` has a command-line option `runtime-namespace`, which is used to specify under which [runtime namespace](https://github.com/containerd/containerd/blob/master/docs/namespaces.md) the particular pod was created. By default, it is set to `k8s.io` and works for containerd when configured +`kata-runtime exec` has a command-line option `runtime-namespace`, which is used to specify under which [runtime namespace](https://github.com/containerd/containerd/blob/main/docs/namespaces.md) the particular pod was created. By default, it is set to `k8s.io` and works for containerd when configured with Kubernetes. For CRI-O, the namespace should set to `default` explicitly. This should not be confused with [Kubernetes namespaces](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/). For other CRI-runtimes and configurations, you may need to set the namespace utilizing the `runtime-namespace` option. From 54d6d01754b689a32da852cfd85a69d2c4730baa Mon Sep 17 00:00:00 2001 From: Ryan Savino Date: Thu, 1 Sep 2022 20:36:28 -0500 Subject: [PATCH 17/26] qemu: fix tdx qemu tarball directories Dockerfile cannot decipher multiple conditional statements in the main RUN call. Cannot segregate statements in Dockerfile with '{}' braces without wrapping entire statement in 'bash -c' statement. Dockerfile does not support setting variables by bash command. Must set HYPERVISOR_NAME and PKGVERSION from parent script: build-base-qemu.sh Fixes: #5078 Signed-Off-By: Ryan Savino --- tools/packaging/static-build/qemu/Dockerfile | 5 ++--- tools/packaging/static-build/qemu/build-base-qemu.sh | 6 +++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/packaging/static-build/qemu/Dockerfile b/tools/packaging/static-build/qemu/Dockerfile index 51b5960477..b06c345504 100644 --- a/tools/packaging/static-build/qemu/Dockerfile +++ b/tools/packaging/static-build/qemu/Dockerfile @@ -56,7 +56,8 @@ ARG QEMU_REPO # commit/tag/branch ARG QEMU_VERSION ARG PREFIX -ARG BUILD_SUFFIX +ARG HYPERVISOR_NAME +ARG PKGVERSION ARG QEMU_DESTDIR ARG QEMU_TARBALL @@ -78,8 +79,6 @@ RUN git clone --depth=1 "${QEMU_REPO}" qemu && \ git fetch --depth=1 origin "${QEMU_VERSION}" && git checkout FETCH_HEAD && \ scripts/git-submodule.sh update meson capstone && \ /root/patch_qemu.sh "${QEMU_VERSION}" "/root/kata_qemu/patches" && \ - [ -n "${BUILD_SUFFIX}" ] && HYPERVISOR_NAME="kata-qemu-${BUILD_SUFFIX}" || HYPERVISOR_NAME="kata-qemu" && \ - [ -n "${BUILD_SUFFIX}" ] && PKGVERSION="kata-static-${BUILD_SUFFIX}" || PKGVERSION="kata-static" && \ (PREFIX="${PREFIX}" /root/configure-hypervisor.sh -s "${HYPERVISOR_NAME}" | xargs ./configure \ --with-pkgversion="${PKGVERSION}") && \ make -j"$(nproc ${CI:+--ignore 1})" && \ diff --git a/tools/packaging/static-build/qemu/build-base-qemu.sh b/tools/packaging/static-build/qemu/build-base-qemu.sh index a16285cd00..7d95f02332 100755 --- a/tools/packaging/static-build/qemu/build-base-qemu.sh +++ b/tools/packaging/static-build/qemu/build-base-qemu.sh @@ -34,9 +34,13 @@ prefix="${prefix:-"/opt/kata"}" CACHE_TIMEOUT=$(date +"%Y-%m-%d") +[ -n "${build_suffix}" ] && HYPERVISOR_NAME="kata-qemu-${build_suffix}" || HYPERVISOR_NAME="kata-qemu" +[ -n "${build_suffix}" ] && PKGVERSION="kata-static-${build_suffix}" || PKGVERSION="kata-static" + sudo "${container_engine}" build \ --build-arg CACHE_TIMEOUT="${CACHE_TIMEOUT}" \ - --build-arg BUILD_SUFFIX="${build_suffix}" \ + --build-arg HYPERVISOR_NAME="${HYPERVISOR_NAME}" \ + --build-arg PKGVERSION="${PKGVERSION}" \ --build-arg http_proxy="${http_proxy}" \ --build-arg https_proxy="${https_proxy}" \ --build-arg QEMU_DESTDIR="${qemu_destdir}" \ From 59e3850bfd02ae497673119871f0a9954b75d685 Mon Sep 17 00:00:00 2001 From: Ryan Savino Date: Thu, 1 Sep 2022 21:07:30 -0500 Subject: [PATCH 18/26] qemu: create no_patches.txt file for SPR-BKC-QEMU-v2.5 Patches failing without the no_patches.txt file for SPR-BKC-QEMU-v2.5. Signed-Off-By: Ryan Savino --- .../qemu/patches/tag_patches/SPR-BKC-QEMU-v2.5/no_patches.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tools/packaging/qemu/patches/tag_patches/SPR-BKC-QEMU-v2.5/no_patches.txt diff --git a/tools/packaging/qemu/patches/tag_patches/SPR-BKC-QEMU-v2.5/no_patches.txt b/tools/packaging/qemu/patches/tag_patches/SPR-BKC-QEMU-v2.5/no_patches.txt new file mode 100644 index 0000000000..e69de29bb2 From 5f4f5f24002b2fe9054cd756dcfb84aff5c5f939 Mon Sep 17 00:00:00 2001 From: Yuan-Zhuo Date: Fri, 2 Sep 2022 09:22:29 +0000 Subject: [PATCH 19/26] docs: fix unix socket address in agent-ctl doc Following the instructions in guidance doc will result in the ECONNREFUSED, thus we need to keep the unix socket address in the two commands consistent. Fixes: #5085 Signed-off-by: Yuan-Zhuo --- src/tools/agent-ctl/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/agent-ctl/README.md b/src/tools/agent-ctl/README.md index d9b53ac35e..667c07b51d 100644 --- a/src/tools/agent-ctl/README.md +++ b/src/tools/agent-ctl/README.md @@ -220,7 +220,7 @@ $ sudo install -o root -g root -m 0755 ~/.cargo/bin/kata-agent-ctl /usr/local/bi 1. Start the agent, specifying a local socket for it to communicate on: ```sh - $ sudo KATA_AGENT_SERVER_ADDR=unix:///tmp/foo.socket target/x86_64-unknown-linux-musl/release/kata-agent + $ sudo KATA_AGENT_SERVER_ADDR=unix://@/tmp/foo.socket target/x86_64-unknown-linux-musl/release/kata-agent ``` > **Note:** This example assumes an Intel x86-64 system. From 373dac2dbb2b0134d12f3ef43a795c594b97bde1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 2 Sep 2022 15:47:48 +0200 Subject: [PATCH 20/26] qemu: Keep passing BUILD_SUFFIX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the commit 54d6d01754b689a32da852cfd85a69d2c4730baa we ended up removing the BUILD_SUFFIX argument passed to QEMU as it only seemed to be used to generate the HYPERVISOR_NAME and PKGVERSION, which were added as arguments to the dockerfile. However, it turns out BUILD_SUFFIX is used by the `qemu-build-post.sh` script, so it can rename the QEMU binary accordingly. Let's just bring it back. Fixes: #5078 Signed-off-by: Fabiano Fidêncio --- tools/packaging/static-build/qemu/Dockerfile | 3 +++ tools/packaging/static-build/qemu/build-base-qemu.sh | 1 + 2 files changed, 4 insertions(+) diff --git a/tools/packaging/static-build/qemu/Dockerfile b/tools/packaging/static-build/qemu/Dockerfile index b06c345504..1e4441daec 100644 --- a/tools/packaging/static-build/qemu/Dockerfile +++ b/tools/packaging/static-build/qemu/Dockerfile @@ -56,6 +56,9 @@ ARG QEMU_REPO # commit/tag/branch ARG QEMU_VERSION ARG PREFIX +# BUILD_SUFFIX is used by the qemu-build-post.sh script to +# properly rename non vanilla versions of the QEMU +ARG BUILD_SUFFIX ARG HYPERVISOR_NAME ARG PKGVERSION ARG QEMU_DESTDIR diff --git a/tools/packaging/static-build/qemu/build-base-qemu.sh b/tools/packaging/static-build/qemu/build-base-qemu.sh index 7d95f02332..0657503f17 100755 --- a/tools/packaging/static-build/qemu/build-base-qemu.sh +++ b/tools/packaging/static-build/qemu/build-base-qemu.sh @@ -39,6 +39,7 @@ CACHE_TIMEOUT=$(date +"%Y-%m-%d") sudo "${container_engine}" build \ --build-arg CACHE_TIMEOUT="${CACHE_TIMEOUT}" \ + --build-arg BUILD_SUFFIX=${build_suffix} \ --build-arg HYPERVISOR_NAME="${HYPERVISOR_NAME}" \ --build-arg PKGVERSION="${PKGVERSION}" \ --build-arg http_proxy="${http_proxy}" \ From e879270a0ca1a35b251997e6431bed88cfed7a29 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Mon, 5 Sep 2022 14:12:06 +0800 Subject: [PATCH 21/26] runtime-rs: add default agent/runtime/hypervisor for configuration Kata 3.0 introduced 3 new configurations under runtime section: name="virt_container" hypervisor_name="dragonball" agent_name="kata" Blank values will lead to starting to fail. Adding default values will make user easy to migrate to kata 3.0. Fixes: #5098 Signed-off-by: Bin Liu --- src/libs/kata-types/src/annotations/mod.rs | 16 ++++++++++++++++ src/libs/kata-types/src/config/agent.rs | 3 +++ src/libs/kata-types/src/config/default.rs | 7 +++++++ src/libs/kata-types/src/config/mod.rs | 4 +++- src/libs/kata-types/src/config/runtime.rs | 3 +++ 5 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index 07cdbe1a1d..1a222bee73 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -12,7 +12,11 @@ use std::u32; use serde::Deserialize; +use crate::config::default::DEFAULT_AGENT_TYPE_NAME; +use crate::config::default::DEFAULT_HYPERVISOR; +use crate::config::default::DEFAULT_RUNTIME_NAME; use crate::config::hypervisor::get_hypervisor_plugin; + use crate::config::TomlConfig; use crate::sl; @@ -439,6 +443,18 @@ impl Annotation { config.runtime.agent_name = ag.to_string(); } } + + // set default values for runtime.name, runtime.hypervisor_name and runtime.agent + if config.runtime.name.is_empty() { + config.runtime.name = DEFAULT_RUNTIME_NAME.to_string() + } + if config.runtime.hypervisor_name.is_empty() { + config.runtime.hypervisor_name = DEFAULT_HYPERVISOR.to_string() + } + if config.runtime.agent_name.is_empty() { + config.runtime.agent_name = DEFAULT_AGENT_TYPE_NAME.to_string() + } + let hypervisor_name = &config.runtime.hypervisor_name; let agent_name = &config.runtime.agent_name; diff --git a/src/libs/kata-types/src/config/agent.rs b/src/libs/kata-types/src/config/agent.rs index c6d23be8ef..fb06de46ce 100644 --- a/src/libs/kata-types/src/config/agent.rs +++ b/src/libs/kata-types/src/config/agent.rs @@ -11,6 +11,9 @@ pub use vendor::AgentVendor; use super::default::{DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT}; +/// agent name of Kata agent. +pub const AGENT_NAME_KATA: &str = "kata"; + /// Kata agent configuration information. #[derive(Debug, Default, Deserialize, Serialize, Clone)] pub struct Agent { diff --git a/src/libs/kata-types/src/config/default.rs b/src/libs/kata-types/src/config/default.rs index 5ba7d07d1a..4bf9e6089d 100644 --- a/src/libs/kata-types/src/config/default.rs +++ b/src/libs/kata-types/src/config/default.rs @@ -6,6 +6,9 @@ //! Default configuration values. #![allow(missing_docs)] +use crate::config::agent::AGENT_NAME_KATA; +use crate::config::hypervisor::HYPERVISOR_NAME_DRAGONBALL; +use crate::config::runtime::RUNTIME_NAME_VIRTCONTAINER; use lazy_static::lazy_static; lazy_static! { @@ -18,6 +21,10 @@ lazy_static! { pub const DEFAULT_AGENT_NAME: &str = "kata-agent"; pub const DEFAULT_AGENT_VSOCK_PORT: u32 = 1024; pub const DEFAULT_AGENT_LOG_PORT: u32 = 1025; +pub const DEFAULT_AGENT_TYPE_NAME: &str = AGENT_NAME_KATA; + +pub const DEFAULT_RUNTIME_NAME: &str = RUNTIME_NAME_VIRTCONTAINER; +pub const DEFAULT_HYPERVISOR: &str = HYPERVISOR_NAME_DRAGONBALL; pub const DEFAULT_INTERNETWORKING_MODEL: &str = "tcfilter"; diff --git a/src/libs/kata-types/src/config/mod.rs b/src/libs/kata-types/src/config/mod.rs index 204adf034b..e837e59c69 100644 --- a/src/libs/kata-types/src/config/mod.rs +++ b/src/libs/kata-types/src/config/mod.rs @@ -29,7 +29,9 @@ pub use self::hypervisor::{ }; mod runtime; -pub use self::runtime::{Runtime, RuntimeVendor}; +pub use self::runtime::{Runtime, RuntimeVendor, RUNTIME_NAME_VIRTCONTAINER}; + +pub use self::agent::AGENT_NAME_KATA; /// Trait to manipulate global Kata configuration information. pub trait ConfigPlugin: Send + Sync { diff --git a/src/libs/kata-types/src/config/runtime.rs b/src/libs/kata-types/src/config/runtime.rs index a9fa3de9db..bfbde60d0d 100644 --- a/src/libs/kata-types/src/config/runtime.rs +++ b/src/libs/kata-types/src/config/runtime.rs @@ -10,6 +10,9 @@ use super::default; use crate::config::{ConfigOps, TomlConfig}; use crate::{eother, resolve_path, validate_path}; +/// Type of runtime VirtContainer. +pub const RUNTIME_NAME_VIRTCONTAINER: &str = "virt_container"; + /// Kata runtime configuration information. #[derive(Debug, Default, Deserialize, Serialize)] pub struct Runtime { From 50f91261532035dce24333bb608cc1089cb687ee Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Tue, 6 Sep 2022 17:32:57 +0800 Subject: [PATCH 22/26] libs/kata-types: replace tabs by spaces in comments Replace tabs by spaces in the comments of file libs/kata-types/src/annotations/mod.rs. Fixes: #5115 Signed-off-by: Bin Liu --- src/libs/kata-types/src/annotations/mod.rs | 28 +++++++++++----------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index 07cdbe1a1d..64d7f22d40 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -177,7 +177,7 @@ pub const KATA_ANNO_CFG_HYPERVISOR_FIRMWARE_HASH: &str = /// A sandbox annotation to specify cpu specific features. pub const KATA_ANNO_CFG_HYPERVISOR_CPU_FEATURES: &str = "io.katacontainers.config.hypervisor.cpu_features"; -/// A sandbox annotation for passing the default vcpus assigned for a VM by the hypervisor. +/// A sandbox annotation for passing the default vCPUs assigned for a VM by the hypervisor. pub const KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS: &str = "io.katacontainers.config.hypervisor.default_vcpus"; /// A sandbox annotation that specifies the maximum number of vCPUs allocated for the VM by the hypervisor. @@ -198,7 +198,7 @@ pub const KATA_ANNO_CFG_HYPERVISOR_IOMMU: &str = "io.katacontainers.config.hyper pub const KATA_ANNO_CFG_HYPERVISOR_IOMMU_PLATFORM: &str = "io.katacontainers.config.hypervisor.enable_iommu_platform"; -// Hypervisor Machine related annotations +// Hypervisor Machine related annotations /// A sandbox annotation to specify the type of machine being emulated by the hypervisor. pub const KATA_ANNO_CFG_HYPERVISOR_MACHINE_TYPE: &str = "io.katacontainers.config.hypervisor.machine_type"; @@ -256,7 +256,7 @@ pub const KATA_ANNO_CFG_HYPERVISOR_ENABLE_ROOTLESS_HYPERVISOR: &str = "io.katacontainers.config.hypervisor.rootless"; // Hypervisor Shared File System related annotations -/// A sandbox annotation to specify the shared file system type, either virtio-9p or virtio-fs. +/// A sandbox annotation to specify the shared file system type, either inline-virtio-fs (default), virtio-9p, virtio-fs or virtio-fs-nydus. pub const KATA_ANNO_CFG_HYPERVISOR_SHARED_FS: &str = "io.katacontainers.config.hypervisor.shared_fs"; /// A sandbox annotations to specify virtio-fs vhost-user daemon path. @@ -452,7 +452,7 @@ impl Annotation { if hv.security_info.is_annotation_enabled(key) { match key.as_str() { // update hypervisor config - // Hypervisor related annotations + // Hypervisor related annotations KATA_ANNO_CFG_HYPERVISOR_PATH => { hv.validate_hypervisor_path(value)?; hv.path = value.to_string(); @@ -475,7 +475,7 @@ impl Annotation { return Err(bool_err); } }, - // Hypervisor Block Device related annotations + // Hypervisor Block Device related annotations KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_DRIVER => { hv.blockdev_info.block_device_driver = value.to_string(); } @@ -571,7 +571,7 @@ impl Annotation { hv.boot_info.validate_boot_path(value)?; hv.boot_info.firmware = value.to_string(); } - // Hypervisor CPU related annotations + // Hypervisor CPU related annotations KATA_ANNO_CFG_HYPERVISOR_CPU_FEATURES => { hv.cpu_info.cpu_features = value.to_string(); } @@ -611,7 +611,7 @@ impl Annotation { } } } - // Hypervisor Device related annotations + // Hypervisor Device related annotations KATA_ANNO_CFG_HYPERVISOR_HOTPLUG_VFIO_ON_ROOT_BUS => { match self.get_value::(key) { Ok(r) => { @@ -646,7 +646,7 @@ impl Annotation { return Err(bool_err); } }, - // Hypervisor Machine related annotations + // Hypervisor Machine related annotations KATA_ANNO_CFG_HYPERVISOR_MACHINE_TYPE => { hv.machine_info.machine_type = value.to_string(); } @@ -657,7 +657,7 @@ impl Annotation { hv.machine_info.validate_entropy_source(value)?; hv.machine_info.entropy_source = value.to_string(); } - // Hypervisor Memory related annotations + // Hypervisor Memory related annotations KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY => { match byte_unit::Byte::from_str(value) { Ok(mem_bytes) => { @@ -746,7 +746,7 @@ impl Annotation { return Err(bool_err); } }, - // Hypervisor Network related annotations + // Hypervisor Network related annotations KATA_ANNO_CFG_HYPERVISOR_DISABLE_VHOST_NET => match self.get_value::(key) { Ok(r) => { @@ -776,7 +776,7 @@ impl Annotation { } } } - // Hypervisor Security related annotations + // Hypervisor Security related annotations KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH => { hv.security_info.validate_path(value)?; hv.security_info.guest_hook_path = value.to_string(); @@ -791,7 +791,7 @@ impl Annotation { } } } - // Hypervisor Shared File System related annotations + // Hypervisor Shared File System related annotations KATA_ANNO_CFG_HYPERVISOR_SHARED_FS => { hv.shared_fs.shared_fs = self.get(key); } @@ -839,7 +839,7 @@ impl Annotation { } } else { match key.as_str() { - //update agent config + // update agent config KATA_ANNO_CFG_KERNEL_MODULES => { let kernel_mod: Vec = value.to_string().split(';').map(str::to_string).collect(); @@ -863,7 +863,7 @@ impl Annotation { return Err(u32_err); } }, - //update runtime config + // update runtime config KATA_ANNO_CFG_RUNTIME_NAME => { let runtime = vec!["virt-container", "linux-container", "wasm-container"]; if runtime.contains(&value.as_str()) { From 188d37badca9c11988b941a688c6a2191bcbe375 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Tue, 6 Sep 2022 13:14:44 -0700 Subject: [PATCH 23/26] kata-deploy: Add debug statement Adding this so that we can see the status of running pods in case of failure. Fixes: #5126 Signed-off-by: Archana Shinde --- tools/packaging/kata-deploy/action/test-kata.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/packaging/kata-deploy/action/test-kata.sh b/tools/packaging/kata-deploy/action/test-kata.sh index 7484ee36dd..96d41fbc61 100755 --- a/tools/packaging/kata-deploy/action/test-kata.sh +++ b/tools/packaging/kata-deploy/action/test-kata.sh @@ -75,7 +75,7 @@ function run_test() { # our 'wait' for deployment status will fail to find the deployment at all sleep 3 - kubectl wait --timeout=5m --for=condition=Available deployment/${deployment} + kubectl wait --timeout=5m --for=condition=Available deployment/${deployment} || kubectl describe pods kubectl expose deployment/${deployment} # test pod connectivity: From d340564d6113b464cb8f70485b14c112928d73a9 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Tue, 6 Sep 2022 14:30:45 -0700 Subject: [PATCH 24/26] Revert "agent: use rtnetlink's neighbours API to add neighbors" This reverts commit 845c1c03cfbe8435ac383964ef243da252726351. Fixes: #5126 --- src/agent/Cargo.lock | 70 +++++++++++++++++++++------------------- src/agent/Cargo.toml | 2 +- src/agent/src/netlink.rs | 60 ++++++++++++++++++++++++++++++---- 3 files changed, 90 insertions(+), 42 deletions(-) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 498a6af2f2..98be055b0f 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -654,8 +654,8 @@ dependencies = [ "libc", "log", "logging", - "netlink-packet-utils 0.4.1", - "netlink-sys 0.7.0", + "netlink-packet-utils", + "netlink-sys", "nix 0.24.2", "oci", "opentelemetry", @@ -841,28 +841,28 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "netlink-packet-core" -version = "0.4.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345b8ab5bd4e71a2986663e88c56856699d060e78e152e6e9d7966fcd5491297" +checksum = "ac48279d5062bdf175bdbcb6b58ff1d6b0ecd54b951f7a0ff4bc0550fe903ccb" dependencies = [ "anyhow", "byteorder", "libc", - "netlink-packet-utils 0.5.1", + "netlink-packet-utils", ] [[package]] name = "netlink-packet-route" -version = "0.13.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5dee5ed749373c298237fe694eb0a51887f4cc1a27370c8464bac4382348f1a" +checksum = "76aed5d3b6e3929713bf1e1334a11fd65180b6d9f5d7c8572664c48b122604f8" dependencies = [ "anyhow", "bitflags", "byteorder", "libc", "netlink-packet-core", - "netlink-packet-utils 0.5.1", + "netlink-packet-utils", ] [[package]] @@ -877,31 +877,19 @@ dependencies = [ "thiserror", ] -[[package]] -name = "netlink-packet-utils" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25af9cf0dc55498b7bd94a1508af7a78706aa0ab715a73c5169273e03c84845e" -dependencies = [ - "anyhow", - "byteorder", - "paste", - "thiserror", -] - [[package]] name = "netlink-proto" -version = "0.10.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65b4b14489ab424703c092062176d52ba55485a89c076b4f9db05092b7223aa6" +checksum = "ddd06e90449ae973fe3888c1ff85949604ef5189b4ac9a2ae39518da1e00762d" dependencies = [ "bytes 1.1.0", "futures", "log", "netlink-packet-core", - "netlink-sys 0.8.3", - "thiserror", + "netlink-sys", "tokio", + "tokio-util", ] [[package]] @@ -917,16 +905,16 @@ dependencies = [ ] [[package]] -name = "netlink-sys" -version = "0.8.3" +name = "nix" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92b654097027250401127914afb37cb1f311df6610a9891ff07a757e94199027" +checksum = "e4916f159ed8e5de0082076562152a76b7a1f64a01fd9d1e0fea002c37624faf" dependencies = [ - "bytes 1.1.0", - "futures", + "bitflags", + "cc", + "cfg-if 1.0.0", "libc", - "log", - "tokio", + "memoffset", ] [[package]] @@ -1491,15 +1479,15 @@ dependencies = [ [[package]] name = "rtnetlink" -version = "0.11.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f1cfa18f8cebe685373a2697915d7e0db3b4554918bba118385e0f71f258a7" +checksum = "7c9a6200d18ec1acfc218ce71363dcc9b6075f399220f903fdfeacd476a876ef" dependencies = [ "futures", "log", "netlink-packet-route", "netlink-proto", - "nix 0.24.2", + "nix 0.22.3", "thiserror", "tokio", ] @@ -1888,6 +1876,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes 1.1.0", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-vsock" version = "0.3.1" diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index 5e045e6340..b0e2ec8cce 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -35,7 +35,7 @@ tokio = { version = "1.14.0", features = ["full"] } tokio-vsock = "0.3.1" netlink-sys = { version = "0.7.0", features = ["tokio_socket",]} -rtnetlink = "0.11.0" +rtnetlink = "0.8.0" netlink-packet-utils = "0.4.1" ipnetwork = "0.17.0" diff --git a/src/agent/src/netlink.rs b/src/agent/src/netlink.rs index 4c14ff6d07..262aaadd73 100644 --- a/src/agent/src/netlink.rs +++ b/src/agent/src/netlink.rs @@ -4,7 +4,7 @@ // use anyhow::{anyhow, Context, Result}; -use futures::{future, TryStreamExt}; +use futures::{future, StreamExt, TryStreamExt}; use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network}; use nix::errno::Errno; use protobuf::RepeatedField; @@ -164,7 +164,7 @@ impl Handle { let request = self.handle.link().get(); let filtered = match filter { - LinkFilter::Name(name) => request.match_name(name.to_owned()), + LinkFilter::Name(name) => request.set_name_filter(name.to_owned()), LinkFilter::Index(index) => request.match_index(index), _ => request, // Post filters }; @@ -516,6 +516,7 @@ impl Handle { } /// Adds an ARP neighbor. + /// TODO: `rtnetlink` has no neighbours API, remove this after https://github.com/little-dude/netlink/pull/135 async fn add_arp_neighbor(&mut self, neigh: &ARPNeighbor) -> Result<()> { let ip_address = neigh .toIPAddress @@ -527,13 +528,58 @@ impl Handle { let ip = IpAddr::from_str(ip_address) .map_err(|e| anyhow!("Failed to parse IP {}: {:?}", ip_address, e))?; + // Import rtnetlink objects that make sense only for this function + use packet::constants::{NDA_UNSPEC, NLM_F_ACK, NLM_F_CREATE, NLM_F_EXCL, NLM_F_REQUEST}; + use packet::neighbour::{NeighbourHeader, NeighbourMessage}; + use packet::nlas::neighbour::Nla; + use packet::{NetlinkMessage, NetlinkPayload, RtnlMessage}; + use rtnetlink::Error; + + const IFA_F_PERMANENT: u16 = 0x80; // See https://github.com/little-dude/netlink/blob/0185b2952505e271805902bf175fee6ea86c42b8/netlink-packet-route/src/rtnl/constants.rs#L770 + let link = self.find_link(LinkFilter::Name(&neigh.device)).await?; - self.handle - .neighbours() - .add(link.index(), ip) - .execute() - .await?; + let message = NeighbourMessage { + header: NeighbourHeader { + family: match ip { + IpAddr::V4(_) => packet::AF_INET, + IpAddr::V6(_) => packet::AF_INET6, + } as u8, + ifindex: link.index(), + state: if neigh.state != 0 { + neigh.state as u16 + } else { + IFA_F_PERMANENT + }, + flags: neigh.flags as u8, + ntype: NDA_UNSPEC as u8, + }, + nlas: { + let mut nlas = vec![Nla::Destination(match ip { + IpAddr::V4(v4) => v4.octets().to_vec(), + IpAddr::V6(v6) => v6.octets().to_vec(), + })]; + + if !neigh.lladdr.is_empty() { + nlas.push(Nla::LinkLocalAddress( + parse_mac_address(&neigh.lladdr)?.to_vec(), + )); + } + + nlas + }, + }; + + // Send request and ACK + let mut req = NetlinkMessage::from(RtnlMessage::NewNeighbour(message)); + req.header.flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE; + + let mut response = self.handle.request(req)?; + while let Some(message) = response.next().await { + if let NetlinkPayload::Error(err) = message.payload { + return Err(anyhow!(Error::NetlinkError(err))); + } + } Ok(()) } From d23779ec9bc7a3b227029212bf37377803f2657a Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Tue, 6 Sep 2022 14:31:33 -0700 Subject: [PATCH 25/26] Revert "agent: fix unittests for arp neighbors" This reverts commit 81fe51ab0b2873edc14d10c95593a65928889e73. --- src/agent/src/netlink.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/agent/src/netlink.rs b/src/agent/src/netlink.rs index 262aaadd73..4d6a26e140 100644 --- a/src/agent/src/netlink.rs +++ b/src/agent/src/netlink.rs @@ -955,7 +955,7 @@ mod tests { .expect("prepare: failed to delete neigh"); } - fn prepare_env_for_test_add_one_arp_neighbor(dummy_name: &str, ip: &str, mac: &str) { + fn prepare_env_for_test_add_one_arp_neighbor(dummy_name: &str, ip: &str) { clean_env_for_test_add_one_arp_neighbor(dummy_name, ip); // modprobe dummy Command::new("modprobe") @@ -969,12 +969,6 @@ mod tests { .output() .expect("failed to add dummy interface"); - // ip link set dummy address 6a:92:3a:59:70:aa - Command::new("ip") - .args(&["link", "set", dummy_name, "address", mac]) - .output() - .expect("failed to add dummy interface"); - // ip addr add 192.168.0.2/16 dev dummy Command::new("ip") .args(&["addr", "add", "192.168.0.2/16", "dev", dummy_name]) @@ -996,7 +990,7 @@ mod tests { let to_ip = "169.254.1.1"; let dummy_name = "dummy_for_arp"; - prepare_env_for_test_add_one_arp_neighbor(dummy_name, to_ip, mac); + prepare_env_for_test_add_one_arp_neighbor(dummy_name, to_ip); let mut ip_address = IPAddress::new(); ip_address.set_address(to_ip.to_string()); From 2e7e81b8d8e30da40b6c2beff0b5cd8756895672 Mon Sep 17 00:00:00 2001 From: Megan Wright Date: Wed, 7 Sep 2022 11:36:29 +0100 Subject: [PATCH 26/26] CCv0: Revert version of tokio in Cargo.lock file Revert version of tokio back to 1.20.1 Signed-off-by: Megan Wright --- src/agent/Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 6265972909..78f227570f 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -3754,9 +3754,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.21.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89797afd69d206ccd11fb0ea560a44bbb87731d020670e79416d442919257d42" +checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581" dependencies = [ "autocfg 1.1.0", "bytes 1.1.0",