From 93ec47092827417b81a2d20c49ea8a4c0fdaae2c Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Tue, 19 Aug 2025 12:23:15 +0200 Subject: [PATCH 1/5] runtime/tests: Update annotation for initdata Let's rename the runtime-rs initdata annotation from `io.katacontainers.config.runtime.cc_init_data` to `io.katacontainers.config.hypervisor.cc_init_data`. Rationale: - initdata itself is a hypervisor-specific feature - the new name aligns with the annotation handling logic: https://github.com/kata-containers/kata-containers/blob/c92bb1aa88fa75005524575de3d192fe747dd3e7/src/libs/kata-types/src/annotations/mod.rs#L514-L968 This commit updates the annotation for go-runtime and tests accordingly. Signed-off-by: Hyounggyu Choi --- src/runtime/pkg/oci/utils_test.go | 3 +++ src/runtime/virtcontainers/pkg/annotations/annotations.go | 2 +- tests/integration/kubernetes/confidential_common.sh | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/runtime/pkg/oci/utils_test.go b/src/runtime/pkg/oci/utils_test.go index 43b3f5fac6..e2e61f7581 100644 --- a/src/runtime/pkg/oci/utils_test.go +++ b/src/runtime/pkg/oci/utils_test.go @@ -885,6 +885,9 @@ func TestAddRuntimeAnnotations(t *testing.T) { runtimeConfig := RuntimeConfig{ HypervisorType: vc.QemuHypervisor, + HypervisorConfig: vc.HypervisorConfig{ + EnableAnnotations: []string{"cc_init_data"}, + }, } ocispec.Annotations[vcAnnotations.DisableGuestSeccomp] = "true" diff --git a/src/runtime/virtcontainers/pkg/annotations/annotations.go b/src/runtime/virtcontainers/pkg/annotations/annotations.go index 03b9e9b70c..3be8a4a086 100644 --- a/src/runtime/virtcontainers/pkg/annotations/annotations.go +++ b/src/runtime/virtcontainers/pkg/annotations/annotations.go @@ -245,7 +245,7 @@ const ( EnableRootlessHypervisor = kataAnnotHypervisorPrefix + "rootless" // Initdata is the initdata passed in when CreateVM - Initdata = kataConfAnnotationsPrefix + "runtime.cc_init_data" + Initdata = kataConfAnnotationsPrefix + "hypervisor.cc_init_data" ) // Runtime related annotations diff --git a/tests/integration/kubernetes/confidential_common.sh b/tests/integration/kubernetes/confidential_common.sh index 617deaf6a0..8df00fe749 100644 --- a/tests/integration/kubernetes/confidential_common.sh +++ b/tests/integration/kubernetes/confidential_common.sh @@ -174,7 +174,7 @@ function create_coco_pod_yaml() { # This function creates pod yaml. Parameters # - $1: image reference # - $2: annotation `io.katacontainers.config.hypervisor.kernel_params` -# - $3: anootation `io.katacontainers.config.runtime.cc_init_data` +# - $3: annotation `io.katacontainers.config.hypervisor.cc_init_data` # - $4: node function create_coco_pod_yaml_with_annotations() { image=$1 @@ -183,7 +183,7 @@ function create_coco_pod_yaml_with_annotations() { node=${4:-} kernel_params_annotation_key="io.katacontainers.config.hypervisor.kernel_params" - cc_initdata_annotation_key="io.katacontainers.config.runtime.cc_init_data" + cc_initdata_annotation_key="io.katacontainers.config.hypervisor.cc_init_data" # Note: this is not local as we use it in the caller test kata_pod="$(new_pod_config "$image" "kata-${KATA_HYPERVISOR}")" From 1f978ecc3163f332c5d3781ca359ba7bd6b17ecf Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Tue, 19 Aug 2025 12:39:24 +0200 Subject: [PATCH 2/5] runtime-rs: Fix issues for empty initdata annotation test Currently, there are 2 issues for the empty initdata annotation test: - Empty string handling - "\[CDH\] \[ERROR\]: Get Resource failed" not appearing `add_hypervisor_initdata_overrides()` does not handle an empty string, which might lead to panic like: ``` called `Result::unwrap()` on an `Err` value: gz decoder failed Caused by: failed to fill whole buffer ``` This commit makes the function return an empty string for a given empty input and updates the assertion string to one that appears in both go-runtime and runtime-rs. Signed-off-by: Alex Lyn Signed-off-by: Hyounggyu Choi --- src/libs/kata-types/src/initdata.rs | 146 +++++++++++++++++- .../integration/kubernetes/k8s-initdata.bats | 2 +- 2 files changed, 144 insertions(+), 4 deletions(-) diff --git a/src/libs/kata-types/src/initdata.rs b/src/libs/kata-types/src/initdata.rs index 65ad341ea3..abf98b781e 100644 --- a/src/libs/kata-types/src/initdata.rs +++ b/src/libs/kata-types/src/initdata.rs @@ -8,6 +8,7 @@ use flate2::read::GzDecoder; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256, Sha384, Sha512}; use std::{collections::HashMap, io::Read}; +use crate::sl; /// Currently, initdata only supports version 0.1.0. const INITDATA_VERSION: &str = "0.1.0"; @@ -203,12 +204,18 @@ pub fn calculate_initdata_digest( Ok(b64encoded_digest) } -/// The argument `initda_annotation` is a Standard base64 encoded string containing a TOML formatted content. +/// The argument `initdata_annotation` is a Standard base64 encoded string containing a TOML formatted content. /// This function decodes the base64 string, parses the TOML content into an InitData structure. -pub fn add_hypervisor_initdata_overrides(initda_annotation: &str) -> Result { +pub fn add_hypervisor_initdata_overrides(initdata_annotation: &str) -> Result { + // If the initdata is empty, return an empty string + if initdata_annotation.is_empty() { + info!(sl!(), "initdata_annotation is empty"); + return Ok("".to_string()); + } + // Base64 decode the annotation value let b64_decoded = - base64::decode_config(initda_annotation, base64::STANDARD).context("base64 decode")?; + base64::decode_config(initdata_annotation, base64::STANDARD).context("base64 decode")?; // Gzip decompress the decoded data let mut gz_decoder = GzDecoder::new(&b64_decoded[..]); @@ -231,6 +238,139 @@ mod tests { use flate2::Compression; use std::io::Write; + // create gzipped and base64 encoded string + fn create_encoded_input(content: &str) -> String { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(content.as_bytes()).unwrap(); + let compressed = encoder.finish().unwrap(); + base64::encode_config(&compressed, base64::STANDARD) + } + + #[test] + fn test_empty_annotation() { + // Test with empty string input + let result = add_hypervisor_initdata_overrides(""); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), ""); + } + + #[test] + fn test_empty_data_section() { + // Test with empty data section + let toml_content = r#" +algorithm = "sha384" +version = "0.1.0" + +[data] +"#; + let encoded = create_encoded_input(toml_content); + + let result = add_hypervisor_initdata_overrides(&encoded); + assert!(result.is_ok()); + } + + #[test] + fn test_valid_complete_initdata() { + // Test with complete InitData structure + let toml_content = r#" +algorithm = "sha384" +version = "0.1.0" + +[data] +"aa.toml" = ''' +[token_configs] +[token_configs.coco_as] +url = 'http://kbs-service.xxx.cluster.local:8080' + +[token_configs.kbs] +url = 'http://kbs-service.xxx.cluster.local:8080' +''' + +"cdh.toml" = ''' +socket = 'unix:///run/guest-services/cdh.sock' +credentials = [] + +[kbc] +name = 'cc_kbc' +url = 'http://kbs-service.xxx.cluster.local:8080' +''' +"#; + let encoded = create_encoded_input(toml_content); + + let result = add_hypervisor_initdata_overrides(&encoded); + assert!(result.is_ok()); + + let output = result.unwrap(); + assert!(!output.is_empty()); + assert!(output.contains("algorithm")); + assert!(output.contains("version")); + } + + #[test] + fn test_invalid_base64() { + // Test with invalid base64 string + let invalid_base64 = "This is not valid base64!"; + + let result = add_hypervisor_initdata_overrides(invalid_base64); + assert!(result.is_err()); + + let error = result.unwrap_err(); + assert!(error.to_string().contains("base64 decode")); + } + + #[test] + fn test_valid_base64_invalid_gzip() { + // Test with valid base64 but invalid gzip content + let not_gzipped = "This is not gzipped content"; + let encoded = base64::encode_config(not_gzipped.as_bytes(), base64::STANDARD); + + let result = add_hypervisor_initdata_overrides(&encoded); + assert!(result.is_err()); + + let error = result.unwrap_err(); + assert!(error.to_string().contains("gz decoder failed")); + } + + #[test] + fn test_missing_algorithm() { + // Test with missing algorithm field + let toml_content = r#" +version = "0.1.0" + +[data] +"test.toml" = ''' +key = "value" +''' +"#; + let encoded = create_encoded_input(toml_content); + + let result = add_hypervisor_initdata_overrides(&encoded); + // This might fail depending on whether algorithm is required + if result.is_err() { + assert!(result.unwrap_err().to_string().contains("parse initdata")); + } + } + + #[test] + fn test_missing_version() { + // Test with missing version field + let toml_content = r#" +algorithm = "sha384" + +[data] +"test.toml" = ''' +key = "value" +''' +"#; + let encoded = create_encoded_input(toml_content); + + let result = add_hypervisor_initdata_overrides(&encoded); + // This might fail depending on whether version is required + if result.is_err() { + assert!(result.unwrap_err().to_string().contains("parse initdata")); + } + } + /// Test InitData creation and serialization #[test] fn test_init_data() { diff --git a/tests/integration/kubernetes/k8s-initdata.bats b/tests/integration/kubernetes/k8s-initdata.bats index 15bcc4993c..1e98cf316a 100644 --- a/tests/integration/kubernetes/k8s-initdata.bats +++ b/tests/integration/kubernetes/k8s-initdata.bats @@ -182,7 +182,7 @@ EOF return 1 fi - assert_logs_contain "${node}" kata "${node_start_time}" "\[CDH\] \[ERROR\]: Get Resource failed" + assert_logs_contain "${node}" kata "${node_start_time}" "\[CDH\] \[ERROR\]: Image Client error: Initialize resource provider failed: Get resource failed" } teardown() { From 208cec429a5dd3eca3d98a3ca358c16dc445330a Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Tue, 19 Aug 2025 11:34:32 +0200 Subject: [PATCH 3/5] runtime-rs: Introduce CoCo-specific enable_annotations We need to include `cc_init_data` in the enable_annotations array to pass the data. Since initdata is a CoCo-specific feature, this commit introduces a new array, `DEFENABLEANNOTATIONS_COCO`, which contains the required string and applies it to the relevant CoCo configuration. Signed-off-by: Hyounggyu Choi --- src/runtime-rs/Makefile | 2 ++ .../config/configuration-qemu-se-runtime-rs.toml.in | 2 +- src/runtime/Makefile | 4 ++-- src/runtime/config/configuration-qemu-coco-dev.toml.in | 2 +- src/runtime/config/configuration-qemu-se.toml.in | 2 +- src/runtime/config/configuration-qemu-tdx.toml.in | 2 +- 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/runtime-rs/Makefile b/src/runtime-rs/Makefile index 7cf88ac042..77cddf4b11 100644 --- a/src/runtime-rs/Makefile +++ b/src/runtime-rs/Makefile @@ -147,6 +147,7 @@ DEFMAXMEMSZ := 0 ##VAR DEFBRIDGES= Default number of bridges DEFBRIDGES := 0 DEFENABLEANNOTATIONS := [\"kernel_params\"] +DEFENABLEANNOTATIONS_COCO := [\"kernel_params\",\"cc_init_data\"] DEFDISABLEGUESTSECCOMP := true DEFDISABLEGUESTEMPTYDIR := false ##VAR DEFAULTEXPFEATURES=[features] Default experimental features enabled @@ -482,6 +483,7 @@ USER_VARS += DEFVIRTIOFSCACHE USER_VARS += DEFVIRTIOFSQUEUESIZE USER_VARS += DEFVIRTIOFSEXTRAARGS USER_VARS += DEFENABLEANNOTATIONS +USER_VARS += DEFENABLEANNOTATIONS_COCO USER_VARS += DEFENABLEIOTHREADS USER_VARS += DEFSECCOMPSANDBOXPARAM USER_VARS += DEFGUESTSELINUXLABEL diff --git a/src/runtime-rs/config/configuration-qemu-se-runtime-rs.toml.in b/src/runtime-rs/config/configuration-qemu-se-runtime-rs.toml.in index a2d6fbecb1..46d1d68647 100644 --- a/src/runtime-rs/config/configuration-qemu-se-runtime-rs.toml.in +++ b/src/runtime-rs/config/configuration-qemu-se-runtime-rs.toml.in @@ -45,7 +45,7 @@ confidential_guest = true # List of valid annotation names for the hypervisor # Each member of the list is a regular expression, which is the base name # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" -enable_annotations = @DEFENABLEANNOTATIONS@ +enable_annotations = @DEFENABLEANNOTATIONS_COCO@ # List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3). diff --git a/src/runtime/Makefile b/src/runtime/Makefile index b10bd0f3f6..748a222c8e 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -217,7 +217,7 @@ DEFMAXMEMSZ := 0 #Default number of bridges DEFBRIDGES := 1 DEFENABLEANNOTATIONS := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\"] -DEFENABLEANNOTATIONSTEE := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"default_vcpus\", \"default_memory\"] +DEFENABLEANNOTATIONS_COCO := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"default_vcpus\", \"default_memory\", \"cc_init_data\"] DEFDISABLEGUESTSECCOMP := true DEFDISABLEGUESTEMPTYDIR := false #Default experimental features enabled @@ -731,7 +731,7 @@ USER_VARS += DEFVIRTIOFSCACHE USER_VARS += DEFVIRTIOFSQUEUESIZE USER_VARS += DEFVIRTIOFSEXTRAARGS USER_VARS += DEFENABLEANNOTATIONS -USER_VARS += DEFENABLEANNOTATIONSTEE +USER_VARS += DEFENABLEANNOTATIONS_COCO USER_VARS += DEFENABLEIOTHREADS USER_VARS += DEFSECCOMPSANDBOXPARAM USER_VARS += DEFENABLEVHOSTUSERSTORE diff --git a/src/runtime/config/configuration-qemu-coco-dev.toml.in b/src/runtime/config/configuration-qemu-coco-dev.toml.in index 79de271708..05ef3b54bb 100644 --- a/src/runtime/config/configuration-qemu-coco-dev.toml.in +++ b/src/runtime/config/configuration-qemu-coco-dev.toml.in @@ -55,7 +55,7 @@ rootfs_type=@DEFROOTFSTYPE@ # List of valid annotation names for the hypervisor # Each member of the list is a regular expression, which is the base name # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" -enable_annotations = @DEFENABLEANNOTATIONSTEE@ +enable_annotations = @DEFENABLEANNOTATIONS_COCO@ # List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3). diff --git a/src/runtime/config/configuration-qemu-se.toml.in b/src/runtime/config/configuration-qemu-se.toml.in index 9f3c258dc8..fba6f4c34d 100644 --- a/src/runtime/config/configuration-qemu-se.toml.in +++ b/src/runtime/config/configuration-qemu-se.toml.in @@ -40,7 +40,7 @@ confidential_guest = true # List of valid annotation names for the hypervisor # Each member of the list is a regular expression, which is the base name # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" -enable_annotations = @DEFENABLEANNOTATIONS@ +enable_annotations = @DEFENABLEANNOTATIONS_COCO@ # List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3). diff --git a/src/runtime/config/configuration-qemu-tdx.toml.in b/src/runtime/config/configuration-qemu-tdx.toml.in index 1c810e8cd6..de8b34c436 100644 --- a/src/runtime/config/configuration-qemu-tdx.toml.in +++ b/src/runtime/config/configuration-qemu-tdx.toml.in @@ -49,7 +49,7 @@ confidential_guest = true # List of valid annotation names for the hypervisor # Each member of the list is a regular expression, which is the base name # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" -enable_annotations = @DEFENABLEANNOTATIONSTEE@ +enable_annotations = @DEFENABLEANNOTATIONS_COCO@ # List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3). From f0db4032f281138e169997e9836a609eab08c25e Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Tue, 19 Aug 2025 10:13:22 +0200 Subject: [PATCH 4/5] Revert "kata-types: Align the initdata annotation with kata-runtime's definition" This reverts commit ede773db176d60aea0a134953f2c3bf05b17c2ea. `cc_init_data` should be under a hypervisor category because it is a hypervisor-specific feature. The annotation including `runtime` also breaks a logic for `is_annotation_enabled()`. Signed-off-by: Hyounggyu Choi --- src/libs/kata-types/src/annotations/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index 8e2f455f17..c7c9975eb2 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -273,7 +273,8 @@ pub const KATA_ANNO_CFG_HYPERVISOR_VIRTIO_FS_EXTRA_ARGS: &str = /// A sandbox annotation to specify as the msize for 9p shares. pub const KATA_ANNO_CFG_HYPERVISOR_MSIZE_9P: &str = "io.katacontainers.config.hypervisor.msize_9p"; /// The initdata annotation passed in when CVM launchs -pub const KATA_ANNO_CFG_RUNTIME_INIT_DATA: &str = "io.katacontainers.config.runtime.cc_init_data"; +pub const KATA_ANNO_CFG_HYPERVISOR_INIT_DATA: &str = + "io.katacontainers.config.hypervisor.cc_init_data"; /// GPU specific annotations for remote hypervisor to help with instance selection /// It's for minimum number of GPUs required for the VM. From 0daafecef21d3c59cdf5a49274f12dfad1a8bca7 Mon Sep 17 00:00:00 2001 From: Hyounggyu Choi Date: Tue, 19 Aug 2025 10:23:38 +0200 Subject: [PATCH 5/5] Revert "runtime-rs: Correct the coresponding initdata annotation const" This reverts commit 37685c41c7f6fceec8634cdd59a863a3b4f8efa7. This renames the relevant constant for initdata. Signed-off-by: Hyounggyu Choi --- src/libs/kata-types/src/annotations/mod.rs | 2 +- src/runtime-rs/crates/hypervisor/src/remote/inner.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libs/kata-types/src/annotations/mod.rs b/src/libs/kata-types/src/annotations/mod.rs index c7c9975eb2..b794b6d8b3 100644 --- a/src/libs/kata-types/src/annotations/mod.rs +++ b/src/libs/kata-types/src/annotations/mod.rs @@ -895,7 +895,7 @@ impl Annotation { hv.security_info.validate_path(value)?; hv.security_info.guest_hook_path = value.to_string(); } - KATA_ANNO_CFG_RUNTIME_INIT_DATA => { + KATA_ANNO_CFG_HYPERVISOR_INIT_DATA => { hv.security_info.initdata = add_hypervisor_initdata_overrides(value).unwrap(); } diff --git a/src/runtime-rs/crates/hypervisor/src/remote/inner.rs b/src/runtime-rs/crates/hypervisor/src/remote/inner.rs index 246c6f19b7..ab6413773d 100644 --- a/src/runtime-rs/crates/hypervisor/src/remote/inner.rs +++ b/src/runtime-rs/crates/hypervisor/src/remote/inner.rs @@ -14,8 +14,8 @@ use kata_types::{ cri_containerd::{SANDBOX_NAMESPACE_LABEL_KEY, SANDBOX_NAME_LABEL_KEY}, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_GPUS, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_GPU_MODEL, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY, KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS, - KATA_ANNO_CFG_HYPERVISOR_IMAGE_PATH, KATA_ANNO_CFG_HYPERVISOR_MACHINE_TYPE, - KATA_ANNO_CFG_RUNTIME_INIT_DATA, + KATA_ANNO_CFG_HYPERVISOR_IMAGE_PATH, KATA_ANNO_CFG_HYPERVISOR_INIT_DATA, + KATA_ANNO_CFG_HYPERVISOR_MACHINE_TYPE, }, capabilities::{Capabilities, CapabilityBits}, }; @@ -127,7 +127,7 @@ impl RemoteInner { config.boot_info.image.to_string(), ); annotations.insert( - KATA_ANNO_CFG_RUNTIME_INIT_DATA.to_string(), + KATA_ANNO_CFG_HYPERVISOR_INIT_DATA.to_string(), config.security_info.initdata.to_string(), ); annotations.insert(